# robots.txt - One World Roofing LLC (CCC1330134)
# https://oneworldroofing.com

User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/
Disallow: /?p=
Disallow: /?page_id=
Allow: /wp-admin/admin-ajax.php

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /*/feed/
Disallow: /?p=
Disallow: /?page_id=
Allow: /wp-admin/admin-ajax.php

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/
Disallow: /xmlrpc.php
Allow: /wp-admin/admin-ajax.php

User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /feed/

User-agent: Baiduspider
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=

# ===========================================================
# SECTION 2 - AI citation crawlers (explicitly allowed)
# These crawlers power AI search answers and citations.
# GPTBot = ChatGPT, PerplexityBot = Perplexity AI,
# ClaudeBot = Claude/Anthropic, OAI-SearchBot = OpenAI web search,
# Amazonbot = Alexa/Amazon AI.
# Google-Extended is a control token for Google AI model use (Gemini,
# Vertex AI); it does not impact inclusion in Google Search.
# ===========================================================

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot
Allow: /
Disallow: /wp-admin/


# ===========================================================
# SECTION 3 - Third-party SEO tool crawlers (disallowed)
# These do not power search results or AI citations.
# ===========================================================

User-agent: DataForSeoBot
Disallow: /

User-agent: RSiteAuditor
Disallow: /


# ===========================================================
# SECTION 4 - Training-only and third-party SEO crawlers
# These do not power search results or AI citations.
# Disallowed to reduce server load with no SEO benefit.
#
# NOTE: blocking AhrefsBot / SemrushBot here does NOT hide our
# backlink profile from competitors - those tools discover links
# by crawling the sites that link TO us, which we do not control.
# It only hides our own page structure. Kept blocked by choice.
# ===========================================================

# Common Crawl - training data harvesting only
User-agent: CCBot
Disallow: /

# FacebookBot is Meta's AI-training crawler and is allowed here.
# Link previews on Facebook use facebookexternalhit, which is a
# separate user agent and is not restricted by this file.
User-agent: FacebookBot
Allow: /

# ByteDance/TikTok AI training
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# All other crawlers
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Allow: /wp-admin/admin-ajax.php

# AI guidance files (not robots directives):
#   https://oneworldroofing.com/llms.txt
#   https://oneworldroofing.com/llms-full.txt

Sitemap: https://oneworldroofing.com/sitemap.xml