# robots.txt for example.com
# Purpose: Block unwanted crawlers, control crawl rate, and declare sitemap(s)
# rewritten 29/06/2025

# ======================
# Block unwanted bots
# ======================

# SemrushBot
User-agent: SemrushBot
Disallow: /

# AhrefsBot
User-agent: AhrefsBot
Disallow: /

# DotBot
User-agent: DotBot
Disallow: /

# MJ12Bot
User-agent: MJ12Bot
Disallow: /

# PetalBot
User-agent: PetalBot
Disallow: /

# BaiduSpider (China)
User-agent: BaiduSpider
Disallow: /

# YandexBot (Russia)
User-agent: YandexBot
Disallow: /

# Sogou Spider (China)
User-agent: Sogou spider
Disallow: /

# ======================
# Throttle crawl rate
# ======================

# Example: Bingbot crawl-delay of 10 seconds
User-agent: Bingbot
Crawl-delay: 10

# ======================
# Allow major search engines
# ======================

User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

# ======================
# Wildcard for all other bots
# ======================

User-agent: *
Disallow:

# ======================
# Sitemap declarations
# ======================

# Production sitemap
#Sitemap: https://www.example.com/sitemap.xml

# Staging or development environment example
# Uncomment below if deploying on staging with basic auth
# User-agent: *
# Disallow: /

# ======================
# End of robots.txt
# ======================