# Robots.txt for listingfeecalc.com
# Updated May 2025

User-agent: *
Allow: /
Disallow: /ad/
Disallow: /*.json$
Disallow: /error
Disallow: /cgi-bin/
Disallow: /tmp/

# Specific directives for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /images/
Allow: /favicon/

User-agent: bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Block archive bots to prevent outdated content being stored
User-agent: ia_archiver
Disallow: /

# Allow social media bots to access image and metadata for rich preview
User-agent: Twitterbot
Allow: /
Allow: /images/

User-agent: facebookexternalhit
Allow: /
Allow: /images/

# Sitemap location
Sitemap: https://listingfeecalc.com/sitemap.xml

# Host directive (helps with canonical domain issues)
Host: listingfeecalc.com