From bbcdd424dac02f60735c89deb1c3df90913d37f3 Mon Sep 17 00:00:00 2001 From: CJSatnarine Date: Fri, 30 Jan 2026 18:50:54 -0500 Subject: [PATCH] add robots.txt file --- templates/robots.txt | 316 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 templates/robots.txt diff --git a/templates/robots.txt b/templates/robots.txt new file mode 100644 index 0000000..2413853 --- /dev/null +++ b/templates/robots.txt @@ -0,0 +1,316 @@ +User-agent: 2^32$ +User-agent: AddSearchBot +User-agent: AdsBot-Google +User-agent: Agentic +User-agent: AhrefsBot +User-agent: .ai +User-agent: AI21 Labs +User-agent: AI2Bot +User-agent: Ai2Bot-Dolma +User-agent: AI2Bot-Dolma +User-agent: AI Article Writer +User-agent: AIBot +User-agent: AI Content Detector +User-agent: AI Dungeon +User-agent: aiHitBot +User-agent: AIMatrix +User-agent: AISearchBot +User-agent: AI Search Engine +User-agent: AI SEO Crawler +User-agent: AI Training +User-agent: AITraining +User-agent: AI Writer +User-agent: Alexa +User-agent: Alpha AI +User-agent: AlphaAI +User-agent: a[mazing]{42}(robot) +User-agent: Amazon Bedrock +User-agent: Amazonbot +User-agent: AmazonBot +User-agent: Amazon Comprehend +User-agent: Amazon-Kendra +User-agent: Amazon Lex +User-agent: Amazon Sagemaker +User-agent: Amazon Silk +User-agent: Amazon Textract +User-agent: Amelia +User-agent: AndersPinkBot +User-agent: Andibot +User-agent: Anthropic +User-agent: anthropic-ai +User-agent: AnyPicker +User-agent: Anyword +User-agent: Applebot +User-agent: Applebot-Extended +User-agent: Aria Browse +User-agent: Articoolo +User-agent: Automated Writer +User-agent: Awario +User-agent: AwarioBot +User-agent: AwarioRssBot +User-agent: AwarioSmartBot +User-agent: Azure +User-agent: BardBot +User-agent: bedrockbot +User-agent: bigsur.ai +User-agent: BLEXBot +User-agent: Brave Leo +User-agent: Brightbot 1.0 +User-agent: ByteDance +User-agent: Bytespider +User-agent: CatBoost +User-agent: CCBot +User-agent: CC-Crawler +User-agent: ChatGLM +User-agent: ChatGPT Agent +User-agent: ChatGPT-User +User-agent: ChatGPT-User/2.0 +User-agent: Chinchilla +User-agent: Claude +User-agent: ClaudeBot +User-agent: Claude-SearchBot +User-agent: Claude-User +User-agent: claude-web +User-agent: Claude-Web +User-agent: ClearScope +User-agent: CloudVertexBot +User-agent: Cohere +User-agent: cohere-ai +User-agent: cohere-training-data-crawler +User-agent: Common Crawl +User-agent: CommonCrawl +User-agent: ContentAtScale +User-agent: ContentBot +User-agent: Contentedge +User-agent: Content Harmony +User-agent: Content King +User-agent: Content Optimizer +User-agent: Content Samurai +User-agent: Conversion AI +User-agent: Copilot +User-agent: CopyAI +User-agent: Copymatic +User-agent: Copyscape +User-agent: Cotoyogi +User-agent: crawler.with.dots +User-agent: CrawlQ AI +User-agent: Crawlspace +User-agent: Crew AI +User-agent: CrewAI +User-agent: curl|sudo bash +User-agent: DALL-E +User-agent: DataForSeoBot +User-agent: DataProvider +User-agent: Datenbank Crawler +User-agent: DeepAI +User-agent: DeepL +User-agent: DeepMind +User-agent: DeepSeek +User-agent: Devin +User-agent: diffbot +User-agent: Diffbot +User-agent: Doubao AI +User-agent: DuckAssistBot +User-agent: Echobot Bot +User-agent: EchoboxBot +User-agent: Facebookbot +User-agent: FacebookBot +User-agent: facebookexternalhit +User-agent: FacebookExternalHit +User-agent: Factset_spyderbot +User-agent: Falcon +User-agent: Firecrawl +User-agent: FirecrawlAgent +User-agent: Flyriver +User-agent: Frase AI +User-agent: FriendlyCrawler +User-agent: Fuzz Faster U Fool +User-agent: Fuzz Faster U Fool v2.0.0 +User-agent: Gemini +User-agent: Gemini-Deep-Research +User-agent: Gemma +User-agent: GenAI +User-agent: Genspark +User-agent: Gigabot +User-agent: GLM +User-agent: GoogleAgent-Mariner +User-agent: Google-CloudVertexBot +User-agent: Google-Extended +User-agent: Google-Firebase +User-agent: GoogleOther +User-agent: GoogleOther-Image +User-agent: GoogleOther-Video +User-agent: Goose +User-agent: GPT +User-agent: GPTBot +User-agent: Grammarly +User-agent: Grendizer +User-agent: Grok +User-agent: GT Bot +User-agent: GTBot +User-agent: Hemingway Editor +User-agent: Hugging Face +User-agent: Hypotenuse AI +User-agent: iaskspider +User-agent: iaskspider/2.0 +User-agent: ICC-Crawler +User-agent: ImageGen +User-agent: ImagesiftBot +User-agent: img2dataset +User-agent: imgproxy +User-agent: Inferkit +User-agent: INK Editor +User-agent: INKforall +User-agent: IntelliSeek +User-agent: ISSCyberRiskCrawler +User-agent: Is this a crawler? +User-agent: JasperAI +User-agent: Kafkai +User-agent: Kangaroo +User-agent: Kangaroo Bot +User-agent: Keyword Density AI +User-agent: Knowledge +User-agent: KomoBot +User-agent: LinerBot +User-agent: LinkedInBot +User-agent: LLaMA +User-agent: LLMs +User-agent: magpie-crawler +User-agent: MarketMuse +User-agent: Meltwater +User-agent: Meta AI +User-agent: Meta-AI +User-agent: MetaAI +User-agent: Meta-External +User-agent: meta-externalagent +User-agent: Meta-ExternalAgent +User-agent: meta-externalfetcher +User-agent: Meta-ExternalFetcher +User-agent: MetaTagBot +User-agent: meta-webindexer +User-agent: Mistral +User-agent: MistralAI-User +User-agent: MistralAI-User/1.0 +User-agent: MJ12bot +User-agent: MyCentralAIScraperBot +User-agent: Narrative +User-agent: NeevaBot +User-agent: netEstate Imprint Crawler +User-agent: NeuralSEO +User-agent: Neural Text +User-agent: Nova Act +User-agent: NovaAct +User-agent: Nutch +User-agent: OAI-SearchBot +User-agent: omgili +User-agent: Omgili +User-agent: omgilibot +User-agent: Omgilibot +User-agent: OmniExplorer_Bot +User-agent: Open AI +User-agent: OpenAI +User-agent: OpenBot +User-agent: OpenText AI +User-agent: Operator +User-agent: Outwrite +User-agent: Page Analyzer AI +User-agent: PanguBot +User-agent: Panscient +User-agent: panscient.com +User-agent: Paperlibot +User-agent: Paraphraser.io +User-agent: peer39_crawler +User-agent: Perplexity +User-agent: PerplexityBot +User-agent: Perplexity-User +User-agent: Petalbot +User-agent: PetalBot +User-agent: Phindbot +User-agent: PhindBot +User-agent: PiplBot +User-agent: Poseidon Research Crawler +User-agent: prefetch-proxy +User-agent: ProWritingAid +User-agent: psbot +User-agent: python-requests +User-agent: QualifiedBot +User-agent: QuillBot +User-agent: quillbot.com +User-agent: RobotSpider +User-agent: Robozilla +User-agent: Rytr +User-agent: SaplingAI +User-agent: SBIntuitionsBot +User-agent: Scalenut +User-agent: Scraper +User-agent: Scrapy +User-agent: ScriptBook +User-agent: Seekr +User-agent: SemrushBot-OCOB +User-agent: SemrushBot-SWA +User-agent: sentibot +User-agent: Sentibot +User-agent: SentiBot +User-agent: SEO Content Machine +User-agent: SEO Robot +User-agent: ShapBot +User-agent: Sidetrade +User-agent: Sidetrade indexer bot +User-agent: Simplified AI +User-agent: Sitefinity +User-agent: Skydancer +User-agent: SlickWrite +User-agent: Sonic +User-agent: Spinbot +User-agent: Spin Rewriter +User-agent: Stability +User-agent: StableDiffusionBot +User-agent: star***crawler +User-agent: Sudowrite +User-agent: SummalyBot +User-agent: Super Agent +User-agent: Surfer AI +User-agent: Teoma +User-agent: TerraCotta +User-agent: Text Blaze +User-agent: TextCortex +User-agent: The Knowledge AI +User-agent: Thinkbot +User-agent: ThinkChaos +User-agent: TikTokSpider +User-agent: Timpibot +User-agent: TimpiBot +User-agent: TurnitinBot +User-agent: VelenPublicWebCrawler +User-agent: Vidnami AI +User-agent: WARDBot +User-agent: Webzio +User-agent: webzio-extended +User-agent: Webzio-Extended +User-agent: Whisper +User-agent: WordAI +User-agent: Wordtune +User-agent: WormsGTP +User-agent: wpbot +User-agent: WPBot +User-agent: Writecream +User-agent: WriterZen +User-agent: Writescope +User-agent: Writesonic +User-agent: xAI +User-agent: xBot +User-agent: YaK +User-agent: YandexAdditional +User-agent: YandexAdditionalBot +User-agent: Youbot +User-agent: YouBot +User-agent: Zerochat +User-agent: Zero GTP +User-agent: Zhipu +User-agent: Zimm +Disallow: / +Disallow: * +DisallowAITraining: / + +Content-Usage: ai=n +Sitemap: {{get_url(path="sitemap.xml")}}