# robots.txt — Rodriguez General Contracting & Custom Painting Inc
# https://rodriguezgeneralcontractinginc.com/
#
# This file is a crawl instruction, not a security control. Anything that
# must stay private is denied at the server level (see .htaccess) — never
# by listing it here, because listing it tells people where to look.

# ---------------------------------------------------------------------------
# Every crawler
# ---------------------------------------------------------------------------
User-agent: *
Allow: /

# Internal search results. Useful to a visitor, worthless to an index, and an
# unlimited supply of near-duplicate query URLs if left crawlable.
Disallow: /search/
Disallow: /search.php
Disallow: /*?q=

# The form's POST target and the page shown after a successful send. Neither
# is a destination anyone should arrive at from a search result.
Disallow: /contact.php
Disallow: /thank-you/

# Application directories. Already blocked at the server, repeated here so a
# misconfigured host does not end up with them indexed.
Disallow: /config/
Disallow: /includes/
Disallow: /components/
Disallow: /pages/
Disallow: /data/
Disallow: /storage/
Disallow: /tools/
Disallow: /vendor/

# ---------------------------------------------------------------------------
# AI and answer-engine crawlers
#
# Explicitly allowed. This is a local trade business: being quotable in an
# AI answer about contractors in San Diego County is the same opportunity as
# ranking in a search result. See llms.txt for the summary written for them.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: CCBot
Allow: /

# ---------------------------------------------------------------------------
# Crawlers that only scrape contact details or hammer the server
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ---------------------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------------------
Sitemap: https://rodriguezgeneralcontractinginc.com/sitemap.xml