# ============================================
# ROBOTS.TXT -- jgs-sa.com
# B2B Product Catalog | WooCommerce
# Optimized Version v5.9 (2026)
# ============================================

# -----------------------------------------------
# SECTION 1: General Rules
# All crawlers (Google, Bing, GPTBot, Perplexity,
# Claude, etc.) unless overridden below
# -----------------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /refer/
Disallow: */trackback/
Disallow: */feed/
Disallow: */embed/

# WooCommerce Pages
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*

# Internal Search
Disallow: /*?s=*

# WooCommerce API — Allow product data (carousel etc.)
Allow: /wp-json/wc/

# Sensitive WooCommerce & WP API endpoints
Disallow: /wp-json/wc/v3/customers
Disallow: /wp-json/wc/v3/orders
Disallow: /wp-json/wc/store/v1/cart
Disallow: /wp-json/wc/store/v1/checkout
Disallow: /wp-json/wp/v2/users

# Faceted navigation — Crawl budget protection
Disallow: /*?orderby=*
Disallow: /*?filter_=*
Disallow: /*?min_price=*
Disallow: /*?max_price=*
Disallow: /*?pa_*

# -----------------------------------------------
# SECTION 2: SEO Tools — Crawl-delay
# -----------------------------------------------
User-agent: AhrefsBot
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wc-api/
Disallow: /*?s=*
Disallow: /wp-json/wc/v3/customers
Disallow: /wp-json/wc/v3/orders
Disallow: /wp-json/wc/store/v1/cart
Disallow: /wp-json/wc/store/v1/checkout
Disallow: /wp-json/wp/v2/users
Crawl-delay: 5

User-agent: SemrushBot
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wc-api/
Disallow: /*?s=*
Disallow: /wp-json/wc/v3/customers
Disallow: /wp-json/wc/v3/orders
Disallow: /wp-json/wc/store/v1/cart
Disallow: /wp-json/wc/store/v1/checkout
Disallow: /wp-json/wp/v2/users
Crawl-delay: 5

# -----------------------------------------------
# SECTION 3: Blocked Bots — AI Scrapers
# (no traffic value, scraping only)
# -----------------------------------------------
User-agent: CCBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# -----------------------------------------------
# SECTION 4: Blocked Bots — Unused SEO Tools
# -----------------------------------------------
User-agent: MJ12bot
Disallow: /

User-agent: Rogerbot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

# -----------------------------------------------
# SITEMAP
# -----------------------------------------------
Sitemap: https://jgs-sa.com/sitemap.xml