User-agent: * Allow: / Disallow: /admin Disallow: /health-intake Disallow: /consent-form Disallow: /quiz-results # ── Block ALL query parameters globally ── Disallow: /*? # ── Old WordPress / WooCommerce paths (no redirects — keep blocked) ── Disallow: /product-category/ Disallow: /page/ Disallow: /wp- Disallow: /wp-content/ Disallow: /wp-includes/ Disallow: /feed/ Disallow: /comments/ Disallow: /cgi-bin/ # ── Old redirecting URLs (REMOVED Disallow — Google must follow 301s) ── # These URLs all have server-side 301 redirects to valid pages. Blocking them # with Disallow prevents Google from following the redirect, which causes the # old URLs to get stuck in "Crawled - currently not indexed" limbo. # Removed: /contact, /about, /contactUs, /aboutUs, /providers, /shop, /product/*, # /cart, /checkout, /learn/*, /category/*, /tag/*, /author/*, /maps/*, /portfolio-category/*, # and all old camelCase / flat test slug paths. # ═══════════════════════════════════════════════ # EXPLICITLY ALLOW PUBLIC CONTENT # ═══════════════════════════════════════════════ Allow: /blog/ Allow: /blog Allow: /tests/ Allow: /tests Allow: /subscriptions Allow: /pharmacogenomics Allow: /brca-test Allow: /hereditary-cancer-test Allow: /newsroom Allow: /our-science Allow: /about-us Allow: /contact-us Allow: /faq Allow: /find-my-test Allow: /compare Allow: /vs-23andme Allow: /testimonials Allow: /franchise Allow: /careers Allow: /service-area Allow: /healthcare-providers Allow: /patients-portal Allow: /providers-portal Allow: /genematrix-app Allow: /genematrix-lims Allow: /rd-innovation Allow: /global-impact Allow: /investor-relations Allow: /gene-matrix-entity Allow: /privacy-policy Allow: /terms-of-service Allow: /hipaa-gdpr Allow: /medical-disclaimer Allow: /dpa Allow: /waiver-of-liability Allow: /return-policy Allow: /accessibility # ═══════════════════════════════════════════════ # GOOGLE BOTS # ═══════════════════════════════════════════════ User-agent: Googlebot Allow: / Disallow: /admin Disallow: /health-intake Disallow: /consent-form Disallow: /quiz-results Disallow: /wp- Disallow: /wp-content/ Disallow: /wp-includes/ Disallow: /feed/ Disallow: /comments/ Disallow: /cgi-bin/ Disallow: /*? Crawl-delay: 2 User-agent: Googlebot-Image Allow: / # Googlebot-Video — blocked because we only use background videos (no watch pages) # This prevents Google Video Search from trying to index decorative hero videos. # Regular web indexing is unaffected — Googlebot still crawls all pages. User-agent: Googlebot-Video Disallow: / User-agent: Googlebot-News Allow: /newsroom Allow: /our-science Allow: /tests/genecancer Allow: /tests/genepgx Allow: /tests/genemind Allow: /rd-innovation Allow: /blog/ # Google AI Overviews & Gemini User-agent: Google-Extended Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # BING / MICROSOFT COPILOT # ═══════════════════════════════════════════════ User-agent: Bingbot Allow: / Disallow: /admin Disallow: /wp- Disallow: /wp-content/ Disallow: /wp-includes/ Disallow: /feed/ Disallow: /comments/ Disallow: /cgi-bin/ Disallow: /*? Crawl-delay: 2 User-agent: msnbot Allow: / Disallow: /admin User-agent: msnbot-media Allow: / # ═══════════════════════════════════════════════ # INTERNATIONAL SEARCH BOTS # ═══════════════════════════════════════════════ User-agent: Baiduspider Allow: / Disallow: /admin User-agent: Baiduspider-image Allow: / User-agent: Baiduspider-video Allow: / User-agent: YandexBot Allow: / Disallow: /admin Disallow: /wp- Disallow: /wp-content/ Disallow: /wp-includes/ Disallow: /feed/ Disallow: /comments/ Disallow: /cgi-bin/ Disallow: /*? Crawl-delay: 2 User-agent: YandexImages Allow: / User-agent: YandexVideo Allow: / User-agent: Yeti Allow: / Disallow: /admin User-agent: NaverBot Allow: / Disallow: /admin User-agent: SeznamBot Allow: / Disallow: /admin User-agent: Sogou Allow: / Disallow: /admin User-agent: Sogou inst spider Allow: / User-agent: Sosospider Allow: / Disallow: /admin User-agent: Teoma Allow: / Disallow: /admin User-agent: Exabot Allow: / Disallow: /admin User-agent: seekingbot Allow: / Disallow: /admin User-agent: rogerbot Allow: / Disallow: /admin User-agent: ia_archiver Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # OPENAI / CHATGPT # ═══════════════════════════════════════════════ User-agent: GPTBot Allow: / Disallow: /admin Crawl-delay: 2 User-agent: ChatGPT-User Allow: / Disallow: /admin User-agent: OAI-SearchBot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # ANTHROPIC / CLAUDE # ═══════════════════════════════════════════════ User-agent: ClaudeBot Allow: / Disallow: /admin User-agent: anthropic-ai Allow: / Disallow: /admin User-agent: Claude-Web Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # PERPLEXITY AI # ═══════════════════════════════════════════════ User-agent: PerplexityBot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # GROK / XAI # ═══════════════════════════════════════════════ User-agent: Grok Allow: / Disallow: /admin User-agent: xAI Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # META AI / LLAMA # ═══════════════════════════════════════════════ User-agent: Meta-ExternalAgent Allow: / Disallow: /admin User-agent: Meta-ExternalFetcher Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # APPLE / SIRI # ═══════════════════════════════════════════════ User-agent: Applebot Allow: / Disallow: /admin User-agent: Applebot-Extended Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # AMAZON ALEXA / AWS # ═══════════════════════════════════════════════ User-agent: Amazonbot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # YOU.COM # ═══════════════════════════════════════════════ User-agent: YouBot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # COHERE AI # ═══════════════════════════════════════════════ User-agent: cohere-ai Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # COMMON CRAWL (AI TRAINING DATA) # ═══════════════════════════════════════════════ User-agent: CCBot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # DIFFBOT (AI KNOWLEDGE GRAPH) # ═══════════════════════════════════════════════ User-agent: Diffbot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # BYTEDANCE / TIKTOK AI # ═══════════════════════════════════════════════ User-agent: Bytespider Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # DUCKDUCKGO # ═══════════════════════════════════════════════ User-agent: DuckDuckBot Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # YAHOO / SLURP # ═══════════════════════════════════════════════ User-agent: Slurp Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # SOCIAL MEDIA CRAWLERS # ═══════════════════════════════════════════════ User-agent: facebookexternalhit Allow: / User-agent: Facebot Allow: / User-agent: LinkedInBot Allow: / User-agent: Twitterbot Allow: / User-agent: X-Crawler Allow: / User-agent: Pinterestbot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / User-agent: Redditbot Allow: / User-agent: Snap URL Preview Service Allow: / User-agent: ToutiaoSpider Allow: / Disallow: /admin # ═══════════════════════════════════════════════ # BLOCK SCRAPERS & SPAM BOTS # ═══════════════════════════════════════════════ User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / # ═══════════════════════════════════════════════ # RSS FEED # ═══════════════════════════════════════════════ Allow: /feed Allow: /rss.xml # ═══════════════════════════════════════════════ # SITEMAPS # ═══════════════════════════════════════════════ Sitemap: https://genematrix.io/sitemap.xml Sitemap: https://genematrix.io/sitemap-news.xml Sitemap: https://genematrix.io/sitemap-products.xml # ═══════════════════════════════════════════════ # HOST DECLARATION # ═══════════════════════════════════════════════ Host: https://genematrix.io