# Global allow for Mediapartners - this is used by Google to place ads in content,
# not indexing purposes.
User-agent: Mediapartners-Google
Disallow:

# Throttles to decrease load
User-agent: Yandex
Crawl-delay: 3

User-agent: Baiduspider
Crawl-delay: 3

User-agent: bingbot
Crawl-delay: 2

User-agent: Slurp
Crawl-delay: 2

User-agent: FacebookBot
Crawl-delay: 3

# Temporary blocks of uninteresting bots
User-Agent: Genieo
Disallow: /

User-Agent: ecoResearch
Disallow: /

User-Agent: Spinn3r
Disallow: /

User-Agent: BLP_bbot
Disallow: /

User-Agent: R6_CommentReader
Disallow: /

User-Agent: omgilibot
Disallow: /

User-Agent: EasouSpider
Disallow: /

User-Agent: A6-Indexer
Disallow: /

User-Agent: magpie-crawler
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: omgili
Disallow: /

# Prevent bots from wasting crawl budget on template/non-content pages
User-Agent: *
Disallow: /search
Disallow: /web
Disallow: /mrcontent
Disallow: /webapi-public
Disallow: /adperfect
Disallow: /*amp.html?*
Disallow: *.ece*

Allow: *.ece*.pdf

# Block indexing of specific sections/content at the market's request
User-Agent: *
Disallow: /advertise/circulars
Disallow: /advertise/classifieds
Disallow: /advertise/welcome-page
Disallow: /advertise/welcome-page/welcome-page-lg
Disallow: /advertise/welcome-page/welcome-page-md
Disallow: /advertise/welcome-page/welcome-page-sm
Disallow: /advertise/welcome-page/welcome-page-xs
Disallow: /customer-service/404
Disallow: /customer-service/410
Disallow: /customer-service/contests-promotions
Disallow: /customer-service/marketing
Disallow: /customer-service/marketing/event
Disallow: /customer-service/marketing/rotary-scholars
Disallow: /customer-service/newsletter-signup
Disallow: /customer-service/plus
Disallow: /customer-service/plus/activate
Disallow: /customer-service/subscription/special-offers/dbm
Disallow: /customer-service/subscription/special-offers/holiday
Disallow: /customer-service/subscription/special-offers/offer-a
Disallow: /customer-service/subscription/special-offers/offer-b
Disallow: /customer-service/subscription/special-offers/offer-c
Disallow: /customer-service/subscription/special-offers/others
Disallow: /customer-service/subscription/special-offers/trial-offer-a
Disallow: /mobile/m-idaho-politics-commentary
Disallow: /site-services/testing/contest-test
Disallow: /site-services/testing/newsgate-testing
Disallow: /site-services/testing/ntl-ad-ops-testing
Disallow: /site-services/testing/PD1021
Disallow: /entertainment/arts-culture/arts-blog
Disallow: /entertainment/books
Disallow: /entertainment/festival
Disallow: /entertainment/holidays
Disallow: /entertainment/local-events
Disallow: /entertainment/movies-news-reviews
Disallow: /entertainment/music-news-reviews
Disallow: /entertainment/tv
Disallow: /living/family
Disallow: /living/food-drink/beer
Disallow: /living/food-drink/wine
Disallow: /living/liv-columns-blogs
Disallow: /living/liv-columns-blogs/advice-columns
Disallow: /living/liv-columns-blogs/tim-woodward
Disallow: /living/liv-columns-blogs/heart-of-treasure-valley/
Disallow: /living/travel
Disallow: /news/business/business-insider
Disallow: /news/business/personal-finance
Disallow: /news/local/community/our-community-blog
Disallow: /news/local/crime/crime-map
Disallow: /news/local/education/in-the-classroom
Disallow: /news/nation-world/health-and-medicine
Disallow: /news/politics-government/politics-columns-blogs
Disallow: /news/politics-government/politics-columns-blogs/politics-blog
Disallow: /news/special-reports
Disallow: /news/special-reports/j-r-simplot-legacy
Disallow: /news/special-reports/larry-craig
Disallow: /opinion/bill-manny
Disallow: /opinion/opn-columns-blogs/robert-ehlert
Disallow: /sports/college/mountain-west/boise-state-university/bronco-beat-blog
Disallow: /sports/college/mountain-west/boise-state-university/m-player-profiles
Disallow: /sports/high-school/varsity-extra-blog
Disallow: /sports/other-sports
Disallow: /sports/scores-stats
Disallow: /sports/spt-columns-blogs
Disallow: /news/politics-government/national-politics
Disallow: /sports/college/football/
Disallow: /sports/nfl
Disallow: /entertainment/ent-columns-blogs
Disallow: /living/home-garden
Disallow: /news/local/community
Disallow: /news/local/news-columns-blogs

Allow: /news/local/news-columns-blogs/helping-works/
Allow: /news/local/news-columns-blogs/letters-from-the-west/
Allow: /news/local/community/boise/
Allow: /news/local/community/canyon-county/
Allow: /news/local/community/west-ada/
Allow: /living/home-garden/margaret-lauterbach/
Allow: /entertainment/ent-columns-blogs/words-deeds/

# Sitemap files
Sitemap: https://www.idahostatesman.com/sitemap/story/update.xml
Sitemap: https://www.idahostatesman.com/sitemap/story/archive.xml
Sitemap: https://www.idahostatesman.com/sitemap/story/googlenews.xml
Sitemap: https://www.idahostatesman.com/sitemap/blogpost/update.xml
Sitemap: https://www.idahostatesman.com/sitemap/blogpost/archive.xml
Sitemap: https://www.idahostatesman.com/sitemap/blogpost/googlenews.xml
Sitemap: https://www.idahostatesman.com/sitemap/section/update.xml
Sitemap: https://www.idahostatesman.com/sitemap/section/archive.xml
Sitemap: https://www.idahostatesman.com/sitemap/topic/archive.xml
Sitemap: https://www.idahostatesman.com/sitemap/topic/update.xml