# robots.txt — howlermag.com
# Howler Magazine — Costa Rica travel, expat & real estate publishing since 1996

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /?s=
Disallow: /search/
# ---------------------------------------------------------------
# CRAWL BLOAT - parameterised duplicate URLs. Added 2026-08-07.
# Query-string variants of pages that already exist at a clean URL.
# No AMP plugin is installed (verified 2026-08-07), so amp/noamp
# are pure duplicates too. Delete this whole block to revert.
# ---------------------------------------------------------------
Disallow: /*?et_blog
Disallow: /*&et_blog
Disallow: /*?*fbclid=
Disallow: /*&fbclid=
Disallow: /*?*utm_
Disallow: /*&utm_
Disallow: /*?*redirect_to=
Disallow: /*?*amp=
Disallow: /*&amp=
Disallow: /*?*noamp=
Disallow: /*&noamp=

# ---------------------------------------------------------------
# GSC cleanup (added 2026-08-19): exact-URL Allows so Googlebot can
# recrawl 18 URLs stuck as "Indexed, though blocked by robots.txt".
# Each URL serves noindex + canonical, so Google drops it after a
# recrawl. Longest-match precedence means these Allows win over the
# wildcard Disallows above for these exact URLs ONLY - the crawl
# bloat block stays in force for everything else. Safe to delete
# this whole block once GSC validation passes.
# ---------------------------------------------------------------
Allow: /category/real-estate/page/4/?amp=1&et_blog
Allow: /fr/travel-adventure-deals/page/70/?et_blog
Allow: /pt/doing-business-in-costa-rica/page/4/?et_blog
Allow: /pura-vida-living-in-cr-2024/page/3/?paged58539=3&et_blog
Allow: /zh-CN/tag/costa-rican-culture/page/4/?et_blog
Allow: /de/category/fashion/page/2/?et_blog
Allow: /de/travel-adventure-deals/page/176/?et_blog
Allow: /fr/travel-adventure-deals/page/30/?et_blog
Allow: /tag/costa-rica/page/10/?et_blog
Allow: /tag/captain-paul-watson/page/2/?amp=1&et_blog
Allow: /es/the-wildlife-of-costa-rica/page/5/?et_blog
Allow: /de/travel-adventure-deals/page/107/?et_blog
Allow: /travel-adventure-deals/page/109/?et_blog
Allow: /iw/fashion/page/3/?et_blog
Allow: /fr/travel-adventure-deals/page/99/?et_blog
Allow: /iw/arts-culture-entertainment-2/page/5/?et_blog
Allow: /it/travel-adventure-deals/page/71/?et_blog
Allow: /category/wildlife/page/16/?et_blog

# ---------------------------------------------------------------
# AI ANSWER ENGINES / RETRIEVAL CRAWLERS — WELCOME.
# These crawlers read our content to answer questions and cite the
# source with a link back. Howler grants permission for this use.
# ---------------------------------------------------------------

User-agent: ClaudeBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

# ---------------------------------------------------------------
# AI MODEL TRAINING CRAWLERS — NOT PERMITTED.
# These crawlers ingest content to train models without attribution.
# ---------------------------------------------------------------

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

# ---------------------------------------------------------------

Sitemap: https://howlermag.com/sitemap.xml

# --- Crawl budget controls (added 5 Aug 2026) ---
# Baidu sends ~2,400 requests/day; negligible audience for an
# English-language Costa Rica publication.
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-render
Disallow: /

# Applebot (search indexing, NOT Applebot-Extended which is
# already blocked above) sent 9,174 requests in one day.
# Slow it down rather than block it - it feeds Siri/Spotlight.
User-agent: Applebot
Crawl-delay: 10