# Robots.txt for RivalSweeper - AI-Powered Competitive Intelligence Platform
# Updated 2026-05-14: Content Signals + Agent discovery links added.
# Policy (2026-10-13): allow AI retrieval bots, Common Crawl, GPTBot and
# Google-Extended so models know RivalSweeper; block scraper/data-harvest bots.

# =============================================================================
# CONTENT SIGNALS (https://contentsignals.org)
# Declare AI content usage preferences. ai-train=yes allows training,
# search=yes allows search indexing, ai-input=yes allows real-time AI answers.
# The Content-Signal line sits inside the User-agent: * group below: outside a
# group, Bing's robots.txt tester reports it as an error.
# =============================================================================

# =============================================================================
# AGENT DISCOVERY ENDPOINTS
# =============================================================================
# Sitemap: https://www.rivalsweeper.com/sitemap.xml (declared at bottom)
# MCP Server Card: https://www.rivalsweeper.com/.well-known/mcp/server-card.json
# Agent Skills: https://www.rivalsweeper.com/.well-known/agent-skills/index.json
# API Catalog: https://www.rivalsweeper.com/.well-known/api-catalog
# OIDC Discovery: https://www.rivalsweeper.com/.well-known/openid-configuration
# ACP Discovery: https://www.rivalsweeper.com/.well-known/acp.json
# Agent Instructions: https://www.rivalsweeper.com/agents.md
# Agentic Sitemap: https://www.rivalsweeper.com/sitemap_agentic_discovery.xml

# =============================================================================
# STANDARD WEB CRAWLERS
# =============================================================================
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /assets/
Disallow: /js/
Disallow: /css/
# Disallow: /images/
Disallow: /fonts/
Disallow: /favicon.ico
Disallow: /favicon2.ico

# Block placeholder/Lorem Ipsum pages that were incorrectly indexed
Disallow: /praesent-libro-se-cursus-ante/
Disallow: /velusce-suscipit-quis-luctus/
Disallow: /torquent-per-conubia-nostra/
Disallow: /sociosqu-ad-litora-torquent/
Disallow: /duis-sagitis-ipsum-prasent/
Disallow: /metus-vitae-pharetra-auctor/

# Block search query URLs
Disallow: /*?s=
Disallow: /*?search=
Disallow: /*?q=

# =============================================================================
# DECOY DISALLOWS (defensive misdirection)
# These paths do NOT exist on this site. We list them so automated scanners
# fingerprint us as a WordPress / Wix / Joomla / Drupal install and waste
# their probe budget hitting paths we never had. Edge WAF returns 444 to all.
# =============================================================================
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /wordpress/
Disallow: /xmlrpc.php
Disallow: /administrator/
Disallow: /joomla/
Disallow: /typo3/
Disallow: /drupal/
Disallow: /sites/default/
Disallow: /_wix/
Disallow: /shopify-admin/
Disallow: /magento/
Disallow: /phpmyadmin/
Disallow: /cpanel/
Disallow: /webmail/
Disallow: /ghost/
Disallow: /sitecore/
Disallow: /umbraco/
Disallow: /bitrix/
Disallow: /_partials/
Disallow: /_api/v2/
Disallow: /squarespace-config/

# =============================================================================
# AI RETRIEVAL BOTS (ALLOW - we want to appear in AI-generated answers)
# These bots fetch content to answer user queries in real-time.
# =============================================================================

# OpenAI: OAI-SearchBot indexes pages for ChatGPT search results (OpenAI's
# crawler docs recommend allowing it explicitly); ChatGPT-User fetches a page
# live for a user's question; GPTBot is OpenAI's model-training crawler, kept
# allowed so base models know RivalSweeper without a live search.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: GPTBot
Allow: /
Crawl-delay: 1

# Anthropic: Claude-SearchBot indexes for Claude search, Claude-User fetches
# live for a user's question, ClaudeBot is the general crawler.
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

# Perplexity AI retrieval
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: Perplexity-User
Allow: /

# Google AI retrieval (GoogleOther is used for AI Overviews / Gemini answers)
User-agent: GoogleOther
Allow: /
Crawl-delay: 1

# Apple Intelligence retrieval
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 1

# Microsoft Bing AI retrieval
User-agent: BingPreview
Allow: /
Crawl-delay: 1

# You.com AI retrieval
User-agent: YouBot
Allow: /
Crawl-delay: 1

# Cohere AI retrieval
User-agent: CohereBot
Allow: /
Crawl-delay: 1

# =============================================================================
# SCRAPER / DATA-HARVEST BOTS (BLOCK)
# These bots scrape content to build/train AI models. Block them.
# =============================================================================



# Anthropic training crawler
User-agent: ClaudeBot-Training
Disallow: /

# Meta/Facebook training crawler
User-agent: FacebookBot
Disallow: /

# ByteDance/TikTok training crawler
User-agent: Bytespider
Disallow: /

# Diffbot (web scraping / training data)
User-agent: Diffbot
Disallow: /

# Omgili / Webz.io data harvesting
User-agent: Omgilibot
Disallow: /

# =============================================================================
# AI MODEL KNOWLEDGE (ALLOW - 2026-10-13: a marketing site wants models to know
# RivalSweeper even without a live search). Common Crawl feeds most LLM
# training sets; Google-Extended controls Gemini training and Gemini app
# grounding.
# =============================================================================

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: Google-Extended
Allow: /

# =============================================================================
# SEARCH ENGINE CRAWLERS (ALLOW)
# =============================================================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /
Crawl-delay: 1

# =============================================================================
# SOCIAL MEDIA CRAWLERS (ALLOW - for link previews)
# =============================================================================

User-agent: Twitterbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 1

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

# =============================================================================
# GENERAL SETTINGS
# =============================================================================


# Sitemap location
Sitemap: https://www.rivalsweeper.com/sitemap.xml
Sitemap: https://www.rivalsweeper.com/sitemap_agentic_discovery.xml