# ----------------------------------------------------------------------- # Primary robots.txt for WordPress + large listing site # Optimized for SEO-safe crawling, image discovery, and reduced junk crawl # ----------------------------------------------------------------------- # ========================= # 1) Block AI / unwanted bots # ========================= User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: ClaudeBot Disallow: / User-agent: GPTBot Disallow: / User-agent: PetalBot Disallow: / User-agent: PerplexityBot Disallow: / User-agent: CCBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: Yandex Disallow: / User-agent: YaK Disallow: / User-agent: EtaoSpider Disallow: / User-agent: viberbot Disallow: / User-agent: uptimerobot Disallow: / # ========================= # 2) Google AI policy # ========================= # Allow Google Search, but block Google AI training/use User-agent: Google-Extended Disallow: / # ========================= # 3) Apple search crawler # ========================= # Allow Apple search indexing, but rate control should be handled at server/WAF level User-agent: Applebot Allow: / # ========================= # 4) Google image crawler # ========================= # Restrict image crawling to the folders/files you actually want surfaced User-agent: Googlebot-Image Disallow: / Allow: /favicon.ico Allow: /wp-content/uploads/ Allow: /images/agents/ Allow: /images/udx/ Allow: /images/listings/ # ========================= # 5) Main search crawlers # ========================= User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # ========================= # 6) General rules for all crawlers # ========================= User-agent: * Allow: / Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Query cleanup / crawl waste reduction Disallow: /*?attachment_id= Disallow: /*?*q= # Faceted / sort-style URLs Disallow: /*/hga Disallow: /*/hga*?*q= Disallow: /*/lga Disallow: /*/lga*?*q= Disallow: /*/oga Disallow: /*/oga*?*q= Disallow: /*/nga Disallow: /*/nga*?*q= # Error / junk patterns Disallow: /*/A Disallow: /*/Array Disallow: /*?*afghanistan* Disallow: /*?*cheapfifa23coins* # Important crawlable pages Allow: /favicon.ico Allow: /en/d/ Allow: /en/d/* Allow: /fr/d/ Allow: /fr/d/* Allow: /nl/d/ Allow: /nl/d/* Allow: /es/d/ Allow: /es/d/* # Account: 1000510 (orig) Disallow: /en/s/ Disallow: /nl/s/ # ------------------------------------------------------- # :: English (EN) Sitemap Search # ------------------------------------------------------- # For Sale Allow: /en/s/for-sale Allow: /en/s/for-sale/condos-apartments Allow: /en/s/for-sale/condos-apartments/pool Allow: /en/s/for-sale/condos-apartments/water-front Allow: /en/s/for-sale/condos-apartments/beach-oceanfront Allow: /en/s/for-sale/condos-apartments/bedrooms-1 Allow: /en/s/for-sale/condos-apartments/bedrooms-2 Allow: /en/s/for-sale/condos-apartments/bedrooms-3 Allow: /en/s/for-sale/single-family-homes Allow: /en/s/for-sale/single-family-homes/pool Allow: /en/s/for-sale/single-family-homes/water-front Allow: /en/s/for-sale/single-family-homes/beach-oceanfront Allow: /en/s/for-sale/single-family-homes/bedrooms-2 Allow: /en/s/for-sale/single-family-homes/bedrooms-3 Allow: /en/s/for-sale/single-family-homes/bedrooms-4 Allow: /en/s/for-sale/single-family-homes/bedrooms-5 # For Rent Allow: /en/s/for-rent Allow: /en/s/for-rent/condos-apartments Allow: /en/s/for-rent/condos-apartments/pool Allow: /en/s/for-rent/condos-apartments/water-front Allow: /en/s/for-rent/condos-apartments/beach-oceanfront Allow: /en/s/for-rent/condos-apartments/bedrooms-1 Allow: /en/s/for-rent/condos-apartments/bedrooms-2 Allow: /en/s/for-rent/condos-apartments/bedrooms-3 Allow: /en/s/for-rent/single-family-homes Allow: /en/s/for-rent/single-family-homes/pool Allow: /en/s/for-rent/single-family-homes/water-front Allow: /en/s/for-rent/single-family-homes/beach-oceanfront Allow: /en/s/for-rent/single-family-homes/bedrooms-2 Allow: /en/s/for-rent/single-family-homes/bedrooms-3 # Land Allow: /en/s/for-sale/vacant-land Allow: /en/s/for-sale/vacant-land/water-front # Commercial Allow: /en/s/commercial # Areas Allow: /en/s/for-sale/alto-vista-noord Allow: /en/s/for-sale/ayo-paradera Allow: /en/s/for-sale/bakval-noord Allow: /en/s/for-sale/balashi-oranjestad-east Allow: /en/s/for-sale/boegeroei-noord Allow: /en/s/for-sale/bubali-noord Allow: /en/s/for-sale/bushiri-oranjestad-west Allow: /en/s/for-sale/calbas-noord Allow: /en/s/for-sale/cashero-santa-cruz Allow: /en/s/for-sale/casibari-paradera Allow: /en/s/for-sale/dakota-oranjestad-east Allow: /en/s/for-sale/eagle-beach-oranjestad-west Allow: /en/s/for-sale/kamay-noord Allow: /en/s/for-sale/koyari-noord Allow: /en/s/for-sale/macuarima-santa-cruz Allow: /en/s/for-sale/malmok-noord Allow: /en/s/for-sale/modanza-oranjestad-east Allow: /en/s/for-sale/other-san-nicolas Allow: /en/s/for-sale/palm-beach-noord Allow: /en/s/for-sale/papaya-paradera Allow: /en/s/for-sale/paradera-paradera Allow: /en/s/for-sale/piedra-plat-paradera Allow: /en/s/for-sale/ponton-oranjestad-west Allow: /en/s/for-sale/pos-abao-oranjestad-west Allow: /en/s/for-sale/pos-chiquito-savaneta Allow: /en/s/for-sale/primavera-oranjestad-east Allow: /en/s/for-sale/renaissance-oranjestad-east Allow: /en/s/for-sale/rooi-afo-oranjestad-east Allow: /en/s/for-sale/rooi-santo-noord Allow: /en/s/for-sale/sabana-basora-savaneta Allow: /en/s/for-sale/sabana-liber-noord Allow: /en/s/for-sale/sabanilla-abou-santa-cruz Allow: /en/s/for-sale/san-fuego-santa-cruz Allow: /en/s/for-sale/sero-alejandro-savaneta Allow: /en/s/for-sale/sero-blanco-oranjestad-west Allow: /en/s/for-sale/seroe-colorado-san-nicolas Allow: /en/s/for-sale/solito-oranjestad-west Allow: /en/s/for-sale/tamarijn-paradera Allow: /en/s/for-sale/tanki-flip-noord Allow: /en/s/for-sale/tanki-leendert-oranjestad-west Allow: /en/s/for-sale/washington-noord Allow: /en/s/for-sale/wayaca-oranjestad-east Allow: /en/s/for-sale/westpunt-noord Allow: /en/s/for-rent/alto-vista-noord Allow: /en/s/for-rent/ayo-paradera Allow: /en/s/for-rent/bakval-noord Allow: /en/s/for-rent/balashi-oranjestad-east Allow: /en/s/for-rent/boegeroei-noord Allow: /en/s/for-rent/bubali-noord Allow: /en/s/for-rent/bushiri-oranjestad-west Allow: /en/s/for-rent/calbas-noord Allow: /en/s/for-rent/cashero-santa-cruz Allow: /en/s/for-rent/casibari-paradera Allow: /en/s/for-rent/dakota-oranjestad-east Allow: /en/s/for-rent/eagle-beach-oranjestad-west Allow: /en/s/for-rent/kamay-noord Allow: /en/s/for-rent/koyari-noord Allow: /en/s/for-rent/macuarima-santa-cruz Allow: /en/s/for-rent/malmok-noord Allow: /en/s/for-rent/modanza-oranjestad-east Allow: /en/s/for-rent/other-san-nicolas Allow: /en/s/for-rent/palm-beach-noord Allow: /en/s/for-rent/papaya-paradera Allow: /en/s/for-rent/paradera-paradera Allow: /en/s/for-rent/piedra-plat-paradera Allow: /en/s/for-rent/ponton-oranjestad-west Allow: /en/s/for-rent/pos-abao-oranjestad-west Allow: /en/s/for-rent/pos-chiquito-savaneta Allow: /en/s/for-rent/primavera-oranjestad-east Allow: /en/s/for-rent/renaissance-oranjestad-east Allow: /en/s/for-rent/rooi-afo-oranjestad-east Allow: /en/s/for-rent/rooi-santo-noord Allow: /en/s/for-rent/sabana-basora-savaneta Allow: /en/s/for-rent/sabana-liber-noord Allow: /en/s/for-rent/sabanilla-abou-santa-cruz Allow: /en/s/for-rent/san-fuego-santa-cruz Allow: /en/s/for-rent/sero-alejandro-savaneta Allow: /en/s/for-rent/sero-blanco-oranjestad-west Allow: /en/s/for-rent/seroe-colorado-san-nicolas Allow: /en/s/for-rent/solito-oranjestad-west Allow: /en/s/for-rent/tamarijn-paradera Allow: /en/s/for-rent/tanki-flip-noord Allow: /en/s/for-rent/tanki-leendert-oranjestad-west Allow: /en/s/for-rent/washington-noord Allow: /en/s/for-rent/wayaca-oranjestad-east Allow: /en/s/for-rent/westpunt-noord # ------------------------------------------------------- # :: Dutch (NL) Sitemap Search # ------------------------------------------------------- # For Sale Allow: /nl/s/te-koop Allow: /nl/s/te-koop/appartementen Allow: /nl/s/te-koop/appartementen/zwembad Allow: /nl/s/te-koop/appartementen/waterkant Allow: /nl/s/te-koop/appartementen/strand-oceanfront Allow: /nl/s/te-koop/appartementen/bedrooms-1 Allow: /nl/s/te-koop/appartementen/bedrooms-2 Allow: /nl/s/te-koop/appartementen/bedrooms-3 Allow: /nl/s/te-koop/huizen Allow: /nl/s/te-koop/huizen/zwembad Allow: /nl/s/te-koop/huizen/waterkant Allow: /nl/s/te-koop/huizen/strand-oceanfront Allow: /nl/s/te-koop/huizen/bedrooms-2 Allow: /nl/s/te-koop/huizen/bedrooms-3 Allow: /nl/s/te-koop/huizen/bedrooms-4 Allow: /nl/s/te-koop/huizen/bedrooms-5 # For Rent Allow: /nl/s/te-huur Allow: /nl/s/te-huur/appartementen Allow: /nl/s/te-huur/appartementen/zwembad Allow: /nl/s/te-huur/appartementen/waterkant Allow: /nl/s/te-huur/appartementen/strand-oceanfront Allow: /nl/s/te-huur/appartementen/bedrooms-1 Allow: /nl/s/te-huur/appartementen/bedrooms-2 Allow: /nl/s/te-huur/appartementen/bedrooms-3 Allow: /nl/s/te-huur/huizen Allow: /nl/s/te-huur/huizen/zwembad Allow: /nl/s/te-huur/huizen/waterkant Allow: /nl/s/te-huur/huizen/strand-oceanfront Allow: /nl/s/te-huur/huizen/bedrooms-2 Allow: /nl/s/te-huur/huizen/bedrooms-3 # Land Allow: /nl/s/te-koop/terrein Allow: /nl/s/te-koop/terrein/waterkant # Commercial Allow: /nl/s/vakantie # Areas Allow: /nl/s/te-koop/alto-vista-noord Allow: /nl/s/te-koop/ayo-paradera Allow: /nl/s/te-koop/bakval-noord Allow: /nl/s/te-koop/balashi-oranjestad-east Allow: /nl/s/te-koop/boegeroei-noord Allow: /nl/s/te-koop/bubali-noord Allow: /nl/s/te-koop/bushiri-oranjestad-west Allow: /nl/s/te-koop/calbas-noord Allow: /nl/s/te-koop/cashero-santa-cruz Allow: /nl/s/te-koop/casibari-paradera Allow: /nl/s/te-koop/dakota-oranjestad-east Allow: /nl/s/te-koop/eagle-beach-oranjestad-west Allow: /nl/s/te-koop/kamay-noord Allow: /nl/s/te-koop/koyari-noord Allow: /nl/s/te-koop/macuarima-santa-cruz Allow: /nl/s/te-koop/malmok-noord Allow: /nl/s/te-koop/modanza-oranjestad-east Allow: /nl/s/te-koop/other-san-nicolas Allow: /nl/s/te-koop/palm-beach-noord Allow: /nl/s/te-koop/papaya-paradera Allow: /nl/s/te-koop/paradera-paradera Allow: /nl/s/te-koop/piedra-plat-paradera Allow: /nl/s/te-koop/ponton-oranjestad-west Allow: /nl/s/te-koop/pos-abao-oranjestad-west Allow: /nl/s/te-koop/pos-chiquito-savaneta Allow: /nl/s/te-koop/primavera-oranjestad-east Allow: /nl/s/te-koop/renaissance-oranjestad-east Allow: /nl/s/te-koop/rooi-afo-oranjestad-east Allow: /nl/s/te-koop/rooi-santo-noord Allow: /nl/s/te-koop/sabana-basora-savaneta Allow: /nl/s/te-koop/sabana-liber-noord Allow: /nl/s/te-koop/sabanilla-abou-santa-cruz Allow: /nl/s/te-koop/san-fuego-santa-cruz Allow: /nl/s/te-koop/sero-alejandro-savaneta Allow: /nl/s/te-koop/sero-blanco-oranjestad-west Allow: /nl/s/te-koop/seroe-colorado-san-nicolas Allow: /nl/s/te-koop/solito-oranjestad-west Allow: /nl/s/te-koop/tamarijn-paradera Allow: /nl/s/te-koop/tanki-flip-noord Allow: /nl/s/te-koop/tanki-leendert-oranjestad-west Allow: /nl/s/te-koop/washington-noord Allow: /nl/s/te-koop/wayaca-oranjestad-east Allow: /nl/s/te-koop/westpunt-noord Allow: /nl/s/te-huur/alto-vista-noord Allow: /nl/s/te-huur/ayo-paradera Allow: /nl/s/te-huur/bakval-noord Allow: /nl/s/te-huur/balashi-oranjestad-east Allow: /nl/s/te-huur/boegeroei-noord Allow: /nl/s/te-huur/bubali-noord Allow: /nl/s/te-huur/bushiri-oranjestad-west Allow: /nl/s/te-huur/calbas-noord Allow: /nl/s/te-huur/cashero-santa-cruz Allow: /nl/s/te-huur/casibari-paradera Allow: /nl/s/te-huur/dakota-oranjestad-east Allow: /nl/s/te-huur/eagle-beach-oranjestad-west Allow: /nl/s/te-huur/kamay-noord Allow: /nl/s/te-huur/koyari-noord Allow: /nl/s/te-huur/macuarima-santa-cruz Allow: /nl/s/te-huur/malmok-noord Allow: /nl/s/te-huur/modanza-oranjestad-east Allow: /nl/s/te-huur/other-san-nicolas Allow: /nl/s/te-huur/palm-beach-noord Allow: /nl/s/te-huur/papaya-paradera Allow: /nl/s/te-huur/paradera-paradera Allow: /nl/s/te-huur/piedra-plat-paradera Allow: /nl/s/te-huur/ponton-oranjestad-west Allow: /nl/s/te-huur/pos-abao-oranjestad-west Allow: /nl/s/te-huur/pos-chiquito-savaneta Allow: /nl/s/te-huur/primavera-oranjestad-east Allow: /nl/s/te-huur/renaissance-oranjestad-east Allow: /nl/s/te-huur/rooi-afo-oranjestad-east Allow: /nl/s/te-huur/rooi-santo-noord Allow: /nl/s/te-huur/sabana-basora-savaneta Allow: /nl/s/te-huur/sabana-liber-noord Allow: /nl/s/te-huur/sabanilla-abou-santa-cruz Allow: /nl/s/te-huur/san-fuego-santa-cruz Allow: /nl/s/te-huur/sero-alejandro-savaneta Allow: /nl/s/te-huur/sero-blanco-oranjestad-west Allow: /nl/s/te-huur/seroe-colorado-san-nicolas Allow: /nl/s/te-huur/solito-oranjestad-west Allow: /nl/s/te-huur/tamarijn-paradera Allow: /nl/s/te-huur/tanki-flip-noord Allow: /nl/s/te-huur/tanki-leendert-oranjestad-west Allow: /nl/s/te-huur/washington-noord Allow: /nl/s/te-huur/wayaca-oranjestad-east Allow: /nl/s/te-huur/westpunt-noord