# we use Shopify as our ecommerce platform # # Robots & Agent policy # --------------------- # Checkouts are for humans. Automated scraping, "buy-for-me" agents, or any # end-to-end flow that completes payment without a final human review step # is not permitted. Legitimate shopping agents must use Shopify's UCP/MCP # endpoints (advertised on each domain at /.well-known/ucp) or the official # Shopify Checkout Kit: https://www.shopify.com/checkout-kit # # Terms of Service: https://www.shopify.com/legal/terms # Contact: bots@shopify.com # # Spec: RFC 9309. Wildcards supported: `*` (zero or more chars), `$` (end of # URL). Character classes (`[a-f]`) are NOT portable — Google, Bing, and the # RFC do not support them. Keep additions to `*` and `$` only. # # Per RFC 9309 §2.2.1, a crawler obeys ONLY the rules in the single most- # specific matching User-agent group — Disallow rules are NOT inherited # from `*`. So any per-bot group below (e.g. AhrefsBot, SemrushBot) must # repeat the same Disallow set, or it is effectively allowed everywhere. # We capture the rules once and reuse them via {{ shared_disallows }}. # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Sitemap: https://www.northernbrewer.com/sitemap.xml User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /services/login_with_shop # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # AI / LLM crawlers # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Policy: # BLOCK bots that scrape content to train foundation models or build # answer-engine indexes by bulk crawling. # ALLOW bots that fetch on behalf of a specific human request (search # citations, link follows, user-initiated browse) — these route # users back to the store and follow the UCP/MCP commerce contract # documented at the top of this file. # # robots.txt is a courtesy, not enforcement. For real protection use # Shopify's edge bot controls or a CDN-level WAF rule. # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ── Disallowed: model training / bulk content scraping ── # OpenAI — training crawler (ChatGPT-User / OAI-SearchBot are allowed below) User-agent: GPTBot Disallow: / # Anthropic — training crawler + legacy browse UA # (Claude-User / Claude-SearchBot are user-initiated and allowed below) User-agent: ClaudeBot Disallow: / User-agent: Claude-Web Disallow: / User-agent: anthropic-ai Disallow: / # Common Crawl — feeds most public training datasets User-agent: CCBot Disallow: / # ByteDance / TikTok User-agent: Bytespider Disallow: / # Google Gemini / Vertex training opt-out (does NOT affect Googlebot search) User-agent: Google-Extended Disallow: / # Apple Intelligence training opt-out (does NOT affect Applebot search) User-agent: Applebot-Extended Disallow: / # Amazon — training + answer engine User-agent: Amazonbot Disallow: / # Meta — Llama training + AI features User-agent: Meta-ExternalAgent Disallow: / User-agent: FacebookBot Disallow: / # Perplexity — bulk index crawler. (Perplexity-User user-initiated browse # is allowed below.) Note: Perplexity has been documented ignoring this # directive; consider WAF enforcement if abuse continues. User-agent: PerplexityBot Disallow: / # Cohere User-agent: cohere-ai Disallow: / User-agent: cohere-training-data-crawler Disallow: / # Allen Institute for AI — Dolma dataset User-agent: AI2Bot Disallow: / User-agent: AI2Bot-Dolma Disallow: / # Diffbot — structured-data extraction sold to AI builders User-agent: Diffbot Disallow: / # Omgili / Webz.io — public-web data brokerage User-agent: Omgili Disallow: / User-agent: Omgilibot Disallow: / User-agent: Webzio-Extended Disallow: / # Image-focused training crawlers User-agent: ImagesiftBot Disallow: / # Huawei User-agent: PetalBot Disallow: / User-agent: PanguBot Disallow: / # You.com User-agent: YouBot Disallow: / # Misc training / dataset crawlers User-agent: Timpibot Disallow: / User-agent: Kangaroo Bot Disallow: / User-agent: Velen Crawler Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: ISSCyberRiskCrawler Disallow: / User-agent: Brightbot 1.0 Disallow: / User-agent: iaskspider/2.0 Disallow: / User-agent: TurnitinBot Disallow: / # Generic scraping framework (often used in custom training pipelines) User-agent: Scrapy Disallow: / # ── Allowed (no rules — documented for clarity): user-initiated agents ── # These bots fetch in response to a human query and link back to the store: # - OAI-SearchBot (ChatGPT search citations) # - ChatGPT-User (ChatGPT browse / link follow) # - Claude-User (Anthropic user-initiated browse) # - Claude-SearchBot (Anthropic search citations) # - Perplexity-User (Perplexity user-initiated browse) # - DuckAssistBot (DuckDuckGo AI assist) # - Applebot (Spotlight / Siri — distinct from Applebot-Extended) # - KagiBot (Kagi search) # - archive.org_bot (Wayback Machine — accountability / journalism) # To switch any of these to "disallowed," add a `User-agent: ` group # followed by `Disallow: /` below. # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # SEO crawlers (rate-limited, not blocked) # Note: Googlebot ignores Crawl-delay — set Google's crawl rate in Search # Console. These directives are honored by Bing, Yandex, Seznam, and most # third-party crawlers. # # Each group repeats the wildcard Disallow set via shared_disallows (see # the spec note at the top of this file) — without it these bots would be # free to crawl /cart, /search, faceted URLs, etc. # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ User-agent: AhrefsBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: AhrefsSiteAudit Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: SemrushBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: SemrushBot-SA Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: MJ12bot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: DotBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: BLEXBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: DataForSeoBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 10 User-agent: Pinterestbot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /27856868/checkouts Disallow: /27856868/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # Faceted-filter / sort crawl traps (Boost Product Filter & Search) Disallow: /collections/*?*_=pf Disallow: /collections/*?*pf_t_ Disallow: /collections/*?*pf_st_ # Add additional pf_ prefixes here if Boost emits them on your store # (pf_p_, pf_v_, pf_n_, pf_c_ are common — verify before adding). # Internal search query results — duplicate-content surface. # (Shopify default already disallows /search; this catches /search?q=…) Disallow: /search?q= # Shopify endpoints not meant for crawlers Disallow: /services/login_with_shop # Auth redirect & return-URL loops Disallow: /*?*return_url= Disallow: /*?*checkout_url= # Shopify share-to-checkout product preview URLs (/{handle}-{hash}-remote) # Anchored with $ so we only catch the share-link suffix, not real products. Disallow: /products/*-remote$ Disallow: /*/products/*-remote$ Disallow: /collections/*/products/*-remote$ Disallow: /*/collections/*/products/*-remote$ Crawl-delay: 1 # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Misbehaving / abusive crawlers — blocked entirely # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ User-agent: Nutch Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: Yeti Disallow: / User-agent: SeekportBot Disallow: /