# Truck Traders — Kyle, TX diesel truck dealership
# We welcome AI/search crawlers. Structured inventory data is published at
# /assets/inventory.json and summarized for LLMs at /llms.txt.
#
# A crawler obeys ONLY its most specific matching group — rules are NOT inherited
# from `User-agent: *`. Every named agent below therefore repeats the admin/api
# disallows. A previous version listed them once in a trailing second
# `User-agent: *` group, which left every named AI crawler free to fetch /admin
# and /api/, and made the file ambiguous by declaring `*` twice.

User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# --- AI answer engines & training crawlers (explicitly allowed) ---
# Consecutive User-agent lines share the one rule group that follows them.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

Sitemap: https://trucktraders.net/sitemap.xml
Sitemap: https://trucktraders.net/sitemap-blog.xml