# Search engines and traffic drivers only User-agent: Googlebot Allow: / Allow: /llms.txt Disallow: /*.jpg$ Disallow: /*.jpeg$ Disallow: /*.png$ Disallow: /*.gif$ Disallow: /*.webp$ Disallow: /*.svg$ Disallow: /*.bmp$ Disallow: /*.tiff$ User-agent: Google-InspectionTool Allow: / User-agent: GoogleOther Allow: / Allow: /llms.txt Disallow: /*.jpg$ Disallow: /*.jpeg$ Disallow: /*.png$ Disallow: /*.gif$ Disallow: /*.webp$ Disallow: /*.svg$ Disallow: /*.bmp$ Disallow: /*.tiff$ User-agent: BingBot Allow: / Allow: /llms.txt Disallow: /*.jpg$ Disallow: /*.jpeg$ Disallow: /*.png$ Disallow: /*.gif$ Disallow: /*.webp$ Disallow: /*.svg$ Disallow: /*.bmp$ Disallow: /*.tiff$ User-agent: DuckDuckBot Allow: / Allow: /llms.txt Disallow: /*.jpg$ Disallow: /*.jpeg$ Disallow: /*.png$ Disallow: /*.gif$ Disallow: /*.webp$ Disallow: /*.svg$ Disallow: /*.bmp$ Disallow: /*.tiff$ # AI inference/citation systems (traffic referring only) User-agent: ChatGPT-User Allow: / Allow: /llms.txt User-agent: ClaudeBot Allow: / Allow: /llms.txt User-agent: PerplexityBot Allow: / Allow: /llms.txt # AI Training Crawlers User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Claude-Web Disallow: / User-agent: Claude-3 Disallow: / User-agent: anthropic Disallow: / User-agent: cohere-ai Disallow: / User-agent: ChatGPT-crawler Disallow: / User-agent: GPTBot-User Disallow: / User-agent: GPTBot Disallow: / User-agent: OpenAI-SearchBot Disallow: / # Social Media Training User-agent: FacebookBot Disallow: / User-agent: Twitterbot Disallow: / User-agent: TikTokBot Disallow: / User-agent: Snapchatbot Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: Meta-ExternalFetcher Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: LinkedInBot Disallow: / # Commercial Scrapers User-agent: WebScraper.io Disallow: / User-agent: Diffbot Disallow: / User-agent: omgilibot Disallow: / User-agent: omgili Disallow: / User-agent: Meltwater Disallow: / User-agent: Seekr Disallow: / # International Search/AI (Training Focus) User-agent: baiduspider Disallow: / User-agent: YandexBot Disallow: / User-agent: YisouSpider Disallow: / User-agent: Sogou Disallow: / User-agent: 360Spider Disallow: / User-agent: SeznamBot Disallow: / User-agent: Bytespider Disallow: / User-agent: YandexGPT Disallow: / # E-commerce/Platform Crawlers User-agent: Amazonbot Disallow: / User-agent: Applebot Disallow: / User-agent: YouBot Disallow: / User-agent: youBot Disallow: / User-agent: Amazon Q Disallow: / # Archive/Research User-agent: ia_archiver Disallow: / User-agent: archive.org_bot Disallow: / # SEO/Marketing Scrapers User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: SemrushBot Allow: / User-agent: MozBot Disallow: / User-agent: DotBot Disallow: / # Additional AI/Bot Crawlers User-agent: meta-externalagent Disallow: / User-agent: Timpibot Disallow: / User-agent: SBIntuitionsBot Disallow: / User-agent: AI2Bot Disallow: / User-agent: Claude-User Disallow: / User-agent: MistralBot Disallow: / User-agent: NeevaAI Disallow: / User-agent: iaskspider Disallow: / User-agent: SearchGPT Disallow: / Sitemap: https://www.servicetarget.com/sitemap.xml