# =========================
# Core web search engines
# =========================
User-agent: Googlebot
Allow: /
User-agent: bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: GoogleOther
Allow: /

# =========================
# AI / Alt search engines
# =========================
# OpenAI / ChatGPT
User-agent: OAI-SearchBot
Allow: /
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Kagi
User-agent: KagiBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# Common Crawl (training-oriented; adjust per policy)
User-agent: CCBot
Disallow: /private/
Allow: /

# AI training controls (not ranking signals)
# Switch to Disallow: / if you want to opt OUT of model training later.
User-agent: Google-Extended
Allow: /
User-agent: Applebot-Extended
Allow: /

# =========================
# Priority crawl areas (discovery)
# =========================
User-agent: *
Allow: /products/
Allow: /collections/
Allow: /blogs/
Allow: /pages/
Allow: /policies/
Allow: /feeds/openai-products.json

# =========================
# Shopify hygiene / noise (reduce duplicates)
# =========================
User-agent: *
# Admin & private
Disallow: /admin
Disallow: /api
Disallow: /sf_private_access_tokens
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /a/downloads/-/*

# Cart / checkout / orders / account
Disallow: /cart
Disallow: /carts
Disallow: /checkout
Disallow: /checkouts/
Disallow: /checkouts.json
Disallow: /orders
Disallow: /account
# Historic paths sometimes seen
Disallow: /82652102954/checkouts
Disallow: /82652102954/orders

# Search & parameter noise (block duplicates, keep core pages indexable)
Disallow: /search
Disallow: /*?*utm_*
Disallow: /*?*variant=*
Disallow: /*?*filter.*
Disallow: /*?*sort_by=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# (We do NOT block /blogs/ or /collections/ paths themselves —
# only noisy query-param versions above)

# App/utility paths
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /tools/recurring
Disallow: /cc

# =========================
# Bot-specific throttles (optional)
# =========================
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: AhrefsSiteAudit
Crawl-delay: 10
User-agent: MJ12bot
Crawl-delay: 10
User-agent: Pinterest
Crawl-delay: 1

# =========================
# Sitemap
# =========================
Sitemap: https://sairahaz.com/sitemap.xml

# Sairahaz SEO crawl-waste controls added 2026-05-26
# Blocks Shopify parameter duplicates that appeared in GSC Crawled - currently not indexed export.
User-agent: *
Disallow: /*?*filter.v.availability=
Disallow: /*?*view=view-
Disallow: /*?*grid_list
Disallow: /*?*_pos=
Disallow: /*?*_fid=
Disallow: /*?*_ss=
Disallow: /*?*pr_rec_id=
Disallow: /*?*pr_rec_pid=
Disallow: /*?*pr_ref_pid=
Disallow: /*?*srsltid=
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /cart/
Disallow: /cdn/