# ==========================================================================
# robots.txt — https://www.offineeds.com
# Updated: 2026-08-24
# Platform: Magento 2 / Adobe Commerce B2B
# Scope: 3,786 product URLs + 137 category URLs + blog + static pages
#
# Objective: keep every commercial and informational URL open to search
# engines AND to AI answer engines, while cutting crawl waste on faceted,
# session, account and system URLs.
#
# RULE THAT DRIVES THIS FILE'S SHAPE:
# a named crawler obeys ONLY the group that names it and ignores the
# "User-agent: *" group completely. So the full disallow set is repeated
# verbatim inside the named group below. Never edit one copy alone.
#
# llms.txt (live, verified): https://www.offineeds.com/llms.txt
# ==========================================================================


# --------------------------------------------------------------------------
# GROUP 1 — DEFAULT: every crawler not named further down
# --------------------------------------------------------------------------
User-agent: *
Allow: /

# Rendering assets — must stay open or Google/AI bots see a broken page
Allow: /media/
Allow: /static/
Allow: /pub/media/
Allow: /pub/static/
Allow: /js/

# Transactional, account and user-generated paths (no search value)
Disallow: /checkout/
Disallow: /customer/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /vendor-pdf-library
Disallow: /sales/
Disallow: /paypal/
Disallow: /newsletter/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /poll/
Disallow: /catalogsearch/
Disallow: /search
Disallow: /gate
Disallow: /cart
Disallow: /checkout
Disallow: /wishlist
Disallow: /index.php/

# Magento system directories
Disallow: /admin/
Disallow: /app/
Disallow: /lib/
Disallow: /var/
Disallow: /setup/
Disallow: /update/
Disallow: /vendor/
Disallow: /generated/
Disallow: /downloader/
Disallow: /errors/
Disallow: /dev/
Disallow: /cron.php

# Session IDs — pattern matches the param anywhere in the query string
Disallow: /*SID=

# Layered navigation + sorting. Confirmed live on /by-category/bag.html:
# material=, price=, product_list_order/dir/limit/mode.
# The rest are Magento 2 defaults, blocked pre-emptively.
Disallow: /*product_list_order=
Disallow: /*product_list_dir=
Disallow: /*product_list_limit=
Disallow: /*product_list_mode=
Disallow: /*material=
Disallow: /*price=
Disallow: /*color=
Disallow: /*size=
Disallow: /*style=
Disallow: /*capacity=

# NOT blocked, on purpose:
#   ?p=      pagination is the crawl path to 3,773 products — keep it open
#   ?utm_*   let Googlebot reach the URL and read rel=canonical, so link
#            equity from tagged backlinks consolidates onto the clean URL


# --------------------------------------------------------------------------
# GROUP 2 — SEARCH ENGINES + AI ANSWER ENGINES (explicitly allowed)
#
# Three jobs in one group, same rules:
#   search crawlers      Googlebot, Bingbot ... classic blue links + AI Overviews
#   AI citation crawlers OAI-SearchBot, Claude-SearchBot, PerplexityBot ...
#                        these build the index AI assistants cite from
#   live fetch agents    ChatGPT-User, Claude-User, Perplexity-User ...
#                        these fetch a page in real time when a buyer asks
#   training/grounding   GPTBot, ClaudeBot, Google-Extended, CCBot ...
#                        allowed so "corporate gifting company India" recalls
#                        the OffiNeeds brand. Catalogue and INR pricing are
#                        already public; nothing here is gated or monetised.
#
# Blocking any citation or live-fetch bot removes OffiNeeds from AI answers
# entirely. That is the single most expensive mistake available in this file.
# --------------------------------------------------------------------------

# Classic search
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
# AI citation / search indexes
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Applebot
User-agent: Amazonbot
# Live user-triggered fetch
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
# Training + AI-grounding permission tokens
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: cohere-ai
Allow: /

Allow: /media/
Allow: /static/
Allow: /pub/media/
Allow: /pub/static/
Allow: /js/

Disallow: /checkout/
Disallow: /customer/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /vendor-pdf-library
Disallow: /sales/
Disallow: /paypal/
Disallow: /newsletter/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /poll/
Disallow: /catalogsearch/
Disallow: /search
Disallow: /gate
Disallow: /cart
Disallow: /checkout
Disallow: /wishlist
Disallow: /index.php/

Disallow: /admin/
Disallow: /app/
Disallow: /lib/
Disallow: /var/
Disallow: /setup/
Disallow: /update/
Disallow: /vendor/
Disallow: /generated/
Disallow: /downloader/
Disallow: /errors/
Disallow: /dev/
Disallow: /cron.php

Disallow: /*SID=
Disallow: /*product_list_order=
Disallow: /*product_list_dir=
Disallow: /*product_list_limit=
Disallow: /*product_list_mode=
Disallow: /*material=
Disallow: /*price=
Disallow: /*color=
Disallow: /*size=
Disallow: /*style=
Disallow: /*capacity=


# --------------------------------------------------------------------------
# GROUP 3 — SEO TOOLING: allowed, throttled
# Needed for OffiNeeds' own audits and rank tracking. Crawl-delay is honoured
# by these vendors (Google ignores it; Google is not in this group).
# --------------------------------------------------------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DataForSeoBot
User-agent: rogerbot
Crawl-delay: 10
Allow: /
Disallow: /checkout/
Disallow: /customer/
Disallow: /cart/
Disallow: /catalogsearch/
Disallow: /search
Disallow: /gate
Disallow: /cart
Disallow: /checkout
Disallow: /wishlist
Disallow: /*SID=
Disallow: /*product_list_order=
Disallow: /*product_list_dir=
Disallow: /*product_list_limit=
Disallow: /*product_list_mode=
Disallow: /*material=
Disallow: /*price=


# --------------------------------------------------------------------------
# GROUP 4 — BLOCKED: bandwidth cost, zero citation or ranking upside
# Bytespider feeds a Chinese-market assistant with no Indian B2B reach.
# ImagesiftBot harvests product photography for training without attribution.
# Diffbot and Omgili resell scraped catalogue data to lead-gen databases.
# The rest are link-graph or content scrapers that never send a buyer.
# --------------------------------------------------------------------------
User-agent: Bytespider
User-agent: ImagesiftBot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Timpibot
User-agent: PanguBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: SeekportBot
User-agent: MegaIndex
User-agent: serpstatbot
Disallow: /


# --------------------------------------------------------------------------
# SITEMAPS
# Root index listed first. The five child sitemaps are listed individually
# because several AI crawlers read a sitemap index without expanding it.
# --------------------------------------------------------------------------
Sitemap: https://www.offineeds.com/sitemap.xml
Sitemap: https://www.offineeds.com/media/sitemap-products.xml
Sitemap: https://www.offineeds.com/media/sitemap-categories.xml
Sitemap: https://www.offineeds.com/media/sitemap-brands.xml
Sitemap: https://www.offineeds.com/media/sitemap-blog.xml
Sitemap: https://www.offineeds.com/media/sitemap-static.xml