add robots.txt file

templates/robots.txt

@@ -0,0 +1,316 @@
+User-agent: 2^32$
+User-agent: AddSearchBot
+User-agent: AdsBot-Google
+User-agent: Agentic
+User-agent: AhrefsBot
+User-agent: .ai
+User-agent: AI21 Labs
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
+User-agent: AI2Bot-Dolma
+User-agent: AI Article Writer
+User-agent: AIBot
+User-agent: AI Content Detector
+User-agent: AI Dungeon
+User-agent: aiHitBot
+User-agent: AIMatrix
+User-agent: AISearchBot
+User-agent: AI Search Engine
+User-agent: AI SEO Crawler
+User-agent: AI Training
+User-agent: AITraining
+User-agent: AI Writer
+User-agent: Alexa
+User-agent: Alpha AI
+User-agent: AlphaAI
+User-agent: a[mazing]{42}(robot)
+User-agent: Amazon Bedrock
+User-agent: Amazonbot
+User-agent: AmazonBot
+User-agent: Amazon Comprehend
+User-agent: Amazon-Kendra
+User-agent: Amazon Lex
+User-agent: Amazon Sagemaker
+User-agent: Amazon Silk
+User-agent: Amazon Textract
+User-agent: Amelia
+User-agent: AndersPinkBot
+User-agent: Andibot
+User-agent: Anthropic
+User-agent: anthropic-ai
+User-agent: AnyPicker
+User-agent: Anyword
+User-agent: Applebot
+User-agent: Applebot-Extended
+User-agent: Aria Browse
+User-agent: Articoolo
+User-agent: Automated Writer
+User-agent: Awario
+User-agent: AwarioBot
+User-agent: AwarioRssBot
+User-agent: AwarioSmartBot
+User-agent: Azure
+User-agent: BardBot
+User-agent: bedrockbot
+User-agent: bigsur.ai
+User-agent: BLEXBot
+User-agent: Brave Leo
+User-agent: Brightbot 1.0
+User-agent: ByteDance
+User-agent: Bytespider
+User-agent: CatBoost
+User-agent: CCBot
+User-agent: CC-Crawler
+User-agent: ChatGLM
+User-agent: ChatGPT Agent
+User-agent: ChatGPT-User
+User-agent: ChatGPT-User/2.0
+User-agent: Chinchilla
+User-agent: Claude
+User-agent: ClaudeBot
+User-agent: Claude-SearchBot
+User-agent: Claude-User
+User-agent: claude-web
+User-agent: Claude-Web
+User-agent: ClearScope
+User-agent: CloudVertexBot
+User-agent: Cohere
+User-agent: cohere-ai
+User-agent: cohere-training-data-crawler
+User-agent: Common Crawl
+User-agent: CommonCrawl
+User-agent: ContentAtScale
+User-agent: ContentBot
+User-agent: Contentedge
+User-agent: Content Harmony
+User-agent: Content King
+User-agent: Content Optimizer
+User-agent: Content Samurai
+User-agent: Conversion AI
+User-agent: Copilot
+User-agent: CopyAI
+User-agent: Copymatic
+User-agent: Copyscape
+User-agent: Cotoyogi
+User-agent: crawler.with.dots
+User-agent: CrawlQ AI
+User-agent: Crawlspace
+User-agent: Crew AI
+User-agent: CrewAI
+User-agent: curl|sudo bash
+User-agent: DALL-E
+User-agent: DataForSeoBot
+User-agent: DataProvider
+User-agent: Datenbank Crawler
+User-agent: DeepAI
+User-agent: DeepL
+User-agent: DeepMind
+User-agent: DeepSeek
+User-agent: Devin
+User-agent: diffbot
+User-agent: Diffbot
+User-agent: Doubao AI
+User-agent: DuckAssistBot
+User-agent: Echobot Bot
+User-agent: EchoboxBot
+User-agent: Facebookbot
+User-agent: FacebookBot
+User-agent: facebookexternalhit
+User-agent: FacebookExternalHit
+User-agent: Factset_spyderbot
+User-agent: Falcon
+User-agent: Firecrawl
+User-agent: FirecrawlAgent
+User-agent: Flyriver
+User-agent: Frase AI
+User-agent: FriendlyCrawler
+User-agent: Fuzz Faster U Fool
+User-agent: Fuzz Faster U Fool v2.0.0
+User-agent: Gemini
+User-agent: Gemini-Deep-Research
+User-agent: Gemma
+User-agent: GenAI
+User-agent: Genspark
+User-agent: Gigabot
+User-agent: GLM
+User-agent: GoogleAgent-Mariner
+User-agent: Google-CloudVertexBot
+User-agent: Google-Extended
+User-agent: Google-Firebase
+User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
+User-agent: Goose
+User-agent: GPT
+User-agent: GPTBot
+User-agent: Grammarly
+User-agent: Grendizer
+User-agent: Grok
+User-agent: GT Bot
+User-agent: GTBot
+User-agent: Hemingway Editor
+User-agent: Hugging Face
+User-agent: Hypotenuse AI
+User-agent: iaskspider
+User-agent: iaskspider/2.0
+User-agent: ICC-Crawler
+User-agent: ImageGen
+User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: imgproxy
+User-agent: Inferkit
+User-agent: INK Editor
+User-agent: INKforall
+User-agent: IntelliSeek
+User-agent: ISSCyberRiskCrawler
+User-agent: Is this a crawler?
+User-agent: JasperAI
+User-agent: Kafkai
+User-agent: Kangaroo
+User-agent: Kangaroo Bot
+User-agent: Keyword Density AI
+User-agent: Knowledge
+User-agent: KomoBot
+User-agent: LinerBot
+User-agent: LinkedInBot
+User-agent: LLaMA
+User-agent: LLMs
+User-agent: magpie-crawler
+User-agent: MarketMuse
+User-agent: Meltwater
+User-agent: Meta AI
+User-agent: Meta-AI
+User-agent: MetaAI
+User-agent: Meta-External
+User-agent: meta-externalagent
+User-agent: Meta-ExternalAgent
+User-agent: meta-externalfetcher
+User-agent: Meta-ExternalFetcher
+User-agent: MetaTagBot
+User-agent: meta-webindexer
+User-agent: Mistral
+User-agent: MistralAI-User
+User-agent: MistralAI-User/1.0
+User-agent: MJ12bot
+User-agent: MyCentralAIScraperBot
+User-agent: Narrative
+User-agent: NeevaBot
+User-agent: netEstate Imprint Crawler
+User-agent: NeuralSEO
+User-agent: Neural Text
+User-agent: Nova Act
+User-agent: NovaAct
+User-agent: Nutch
+User-agent: OAI-SearchBot
+User-agent: omgili
+User-agent: Omgili
+User-agent: omgilibot
+User-agent: Omgilibot
+User-agent: OmniExplorer_Bot
+User-agent: Open AI
+User-agent: OpenAI
+User-agent: OpenBot
+User-agent: OpenText AI
+User-agent: Operator
+User-agent: Outwrite
+User-agent: Page Analyzer AI
+User-agent: PanguBot
+User-agent: Panscient
+User-agent: panscient.com
+User-agent: Paperlibot
+User-agent: Paraphraser.io
+User-agent: peer39_crawler
+User-agent: Perplexity
+User-agent: PerplexityBot
+User-agent: Perplexity-User
+User-agent: Petalbot
+User-agent: PetalBot
+User-agent: Phindbot
+User-agent: PhindBot
+User-agent: PiplBot
+User-agent: Poseidon Research Crawler
+User-agent: prefetch-proxy
+User-agent: ProWritingAid
+User-agent: psbot
+User-agent: python-requests
+User-agent: QualifiedBot
+User-agent: QuillBot
+User-agent: quillbot.com
+User-agent: RobotSpider
+User-agent: Robozilla
+User-agent: Rytr
+User-agent: SaplingAI
+User-agent: SBIntuitionsBot
+User-agent: Scalenut
+User-agent: Scraper
+User-agent: Scrapy
+User-agent: ScriptBook
+User-agent: Seekr
+User-agent: SemrushBot-OCOB
+User-agent: SemrushBot-SWA
+User-agent: sentibot
+User-agent: Sentibot
+User-agent: SentiBot
+User-agent: SEO Content Machine
+User-agent: SEO Robot
+User-agent: ShapBot
+User-agent: Sidetrade
+User-agent: Sidetrade indexer bot
+User-agent: Simplified AI
+User-agent: Sitefinity
+User-agent: Skydancer
+User-agent: SlickWrite
+User-agent: Sonic
+User-agent: Spinbot
+User-agent: Spin Rewriter
+User-agent: Stability
+User-agent: StableDiffusionBot
+User-agent: star***crawler
+User-agent: Sudowrite
+User-agent: SummalyBot
+User-agent: Super Agent
+User-agent: Surfer AI
+User-agent: Teoma
+User-agent: TerraCotta
+User-agent: Text Blaze
+User-agent: TextCortex
+User-agent: The Knowledge AI
+User-agent: Thinkbot
+User-agent: ThinkChaos
+User-agent: TikTokSpider
+User-agent: Timpibot
+User-agent: TimpiBot
+User-agent: TurnitinBot
+User-agent: VelenPublicWebCrawler
+User-agent: Vidnami AI
+User-agent: WARDBot
+User-agent: Webzio
+User-agent: webzio-extended
+User-agent: Webzio-Extended
+User-agent: Whisper
+User-agent: WordAI
+User-agent: Wordtune
+User-agent: WormsGTP
+User-agent: wpbot
+User-agent: WPBot
+User-agent: Writecream
+User-agent: WriterZen
+User-agent: Writescope
+User-agent: Writesonic
+User-agent: xAI
+User-agent: xBot
+User-agent: YaK
+User-agent: YandexAdditional
+User-agent: YandexAdditionalBot
+User-agent: Youbot
+User-agent: YouBot
+User-agent: Zerochat
+User-agent: Zero GTP
+User-agent: Zhipu
+User-agent: Zimm
+Disallow: /
+Disallow: *
+DisallowAITraining: /
+
+Content-Usage: ai=n
+Sitemap: {{get_url(path="sitemap.xml")}}