{ "schema_version": "2026-07-23", "last_reviewed": "2026-07-23", "public_scope": "Machine-readable explanation of the current robots.txt posture for search, citation, and AI crawlers. robots.txt remains authoritative.", "authoritative_policy_url": "https://www.touchclass.com/robots.txt", "current_robots_policy": { "default_public_pages": "allowed", "blocked_paths": [ "/include/", "/canvas/", "/api/", "/cache/", "/newin_gitpull.php" ], "machine_readable_surfaces_explicitly_crawlable": [ "https://www.touchclass.com/llms.txt", "https://www.touchclass.com/llms-full.txt", "https://www.touchclass.com/ko/llms.txt", "https://www.touchclass.com/ko/llms-full.txt", "https://www.touchclass.com/en/llms.txt", "https://www.touchclass.com/en/llms-full.txt", "https://www.touchclass.com/ja/llms.txt", "https://www.touchclass.com/ja/llms-full.txt", "https://www.touchclass.com/data/ (all public JSON)", "https://www.touchclass.com/markdown.php/{ko,en,ja}/{page}.md (page mirrors served as text/markdown)", "https://www.touchclass.com/csv.php/{asset}.csv (public CSV assets served as text/csv)", "https://www.touchclass.com/{ko,en,ja}/*.md (legacy static Markdown mirrors)" ], "search_crawlers_allowed": [ "Googlebot", "Googlebot-Image", "Bingbot", "msnbot", "Yeti" ], "ai_search_or_user_crawlers_allowed": [ "OAI-SearchBot", "ChatGPT-User", "OAI-AdsBot", "PerplexityBot", "Perplexity-User", "Claude-SearchBot", "Claude-User", "Claude-Code", "Applebot", "DuckAssistBot", "Amazonbot", "Amzn-User", "GoogleOther", "Google-CloudVertexBot", "Google-NotebookLM", "MistralAI-User", "cohere-ai", "YouBot", "ExaBot", "TavilyBot", "LinerBot", "PhindBot", "Bravebot", "kagi-fetcher", "YandexAdditional", "WRTNBot", "SBIntuitionsBot", "ICC-Crawler", "Cotoyogi", "PetalBot", "TongyiBot", "Kimi-User" ], "ads_crawlers_allowed": [ "AdsBot-Google", "AdsBot-Google-Mobile" ], "training_or_control_tokens_currently_allowed": [ "GPTBot", "ClaudeBot", "Google-Extended" ] }, "changes_2026_07_14": [ "Removed Crawl-delay from Bingbot, PerplexityBot, Claude-SearchBot, and ClaudeBot. Bing honors Crawl-delay, and Bingbot crawl latency propagates into ChatGPT search index freshness. With 599 total URLs there is no load justification. Crawl-delay is retained for Yeti (Naver) pending a load review.", "Added an explicit group for AdsBot-Google and AdsBot-Google-Mobile. Google documents that AdsBot ignores the global `*` user agent, so without an explicit group these crawlers had no applicable rules.", "Added explicit Allow groups for answer-engine and AI-search crawlers so that a future tightening of the `*` group cannot silently cut off citation traffic.", "Did NOT add Gemini-Deep-Research or GoogleAgent-URLContext: neither token appears in Google's official crawler documentation as of 2026-07-14.", "Did NOT add 'ChatGPT Agent' or 'Operator' as tokens: robots.txt product tokens cannot contain spaces (RFC 9309), and OpenAI routes agent/Operator traffic through the already-allowed ChatGPT-User token." ], "changes_2026_07_23": [ "Added a public /markdown.php representation for generated page mirrors. It serves the existing mirror bytes as text/markdown with language, ETag, Last-Modified, canonical Link, noindex, CORS, and nosniff headers.", "Changed llms.txt catalogs, Markdown indexes, and page-level alternate discovery links to advertise the correct-MIME endpoint while retaining the static .md files as generated source artifacts.", "Kept search sitemap URLs HTML-only; machine representations are noindex alternatives and are not separate search results.", "Added an allowlisted /csv.php representation for five public generated CSV assets. It serves the existing data files as text/csv with ETag, Last-Modified, noindex, CORS, and cache headers while retaining the static /data/*.csv paths as compatibility artifacts." ], "token_verification": { "verified_against_official_docs_2026_07_14": { "Googlebot, Googlebot-Image, GoogleOther, Google-Extended, Google-CloudVertexBot": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "Google-NotebookLM": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "AdsBot-Google, AdsBot-Google-Mobile": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers", "OAI-SearchBot, ChatGPT-User, OAI-AdsBot, GPTBot": "https://developers.openai.com/api/docs/bots", "Applebot, Applebot-Extended": "https://support.apple.com/en-us/119829", "DuckAssistBot": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/", "Amazonbot": "https://developer.amazon.com/amazonbot", "CCBot": "https://commoncrawl.org/ccbot" }, "allowed_but_not_yet_doc_verified": { "tokens": [ "Amzn-User", "MistralAI-User", "cohere-ai", "YouBot", "ExaBot", "TavilyBot", "LinerBot", "PhindBot", "Bravebot", "kagi-fetcher", "YandexAdditional", "Claude-Code", "WRTNBot", "SBIntuitionsBot", "ICC-Crawler", "Cotoyogi", "PetalBot", "TongyiBot", "Kimi-User", "msnbot" ], "risk": "none", "why_no_risk": "These appear only in Allow-only groups. If a token string is inaccurate it simply never matches, and the crawler falls back to the permissive `*` group. No access is lost.", "action": "Confirm each against its vendor's official crawler documentation on the next review." }, "verified_against_official_docs_2026_08_21": { "OAI-SearchBot, GPTBot, OAI-AdsBot, ChatGPT-User": "https://developers.openai.com/api/docs/bots", "ClaudeBot, Claude-User, Claude-SearchBot": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "PerplexityBot, Perplexity-User": "https://docs.perplexity.ai/guides/bots" } }, "pending_business_decision": { "question": "Should pure training/dataset crawlers be allowed to keep collecting TouchClass content?", "current_state": "ALLOWED BY DEFAULT — not an explicit decision. These crawlers are covered by the permissive `User-agent: *` group.", "affected_tokens": [ "CCBot", "Applebot-Extended", "meta-externalagent", "Bytespider", "Diffbot", "omgili", "Timpibot" ], "note_on_applebot_extended": "Applebot-Extended is a training opt-out control token, not a crawler. Disallowing it stops Apple from using already-crawled content for model training; it does NOT reduce Applebot search/Siri visibility.", "decoupling_note": "Blocking training crawlers does NOT reduce citation visibility in answer engines (OAI-SearchBot, Claude-SearchBot, PerplexityBot, etc.). The two axes can be decided independently.", "owner": "business, legal, security, and marketing decision", "escalation": "Report to CEO that these are currently allowed by default, then fix the chosen default as an explicit robots.txt group." }, "openai_bots_reference": { "docs": "https://developers.openai.com/api/docs/bots", "OAI-SearchBot": { "purpose": "ChatGPT search indexing", "ip_ranges": "https://openai.com/searchbot.json", "robots_effect_delay": "~24h" }, "ChatGPT-User": { "purpose": "user-triggered live fetch inside ChatGPT, including agent and Operator traffic", "ip_ranges": "https://openai.com/chatgpt-user.json" }, "OAI-AdsBot": { "purpose": "ad landing page validation", "ip_ranges": "https://openai.com/adsbot.json" }, "GPTBot": { "purpose": "generative model training crawl", "ip_ranges": "https://openai.com/gptbot.json" } }, "known_infrastructure_issue": { "issue": "Production nginx serves legacy static .md page mirrors and public .csv assets as application/octet-stream with nosniff, so crawlers see them as binary downloads rather than readable text.", "impact": "Only direct requests to the legacy static paths are affected. Advertised machine links now use the correct-MIME Markdown and CSV streaming endpoints.", "status": "Mitigated in application code on 2026-07-23. An nginx MIME change remains optional cleanup for legacy static URLs.", "fix_doc": "docs/ops/prod-nginx-md-mimetype.md", "verify_command": "curl -sSI -A 'OAI-SearchBot' 'https://www.touchclass.com/markdown.php/ko/price.md' | grep -i content-type; curl -sSI -A 'OAI-SearchBot' 'https://www.touchclass.com/csv.php/lms-implementation-checklist.csv' | grep -i content-type" }, "recommended_default": { "search_and_citation_crawlers": "allow public marketing pages and machine-readable public files", "private_or_operational_paths": "block", "training_bot_policy": "business_decision_required" }, "claim_cautions": [ "Do not describe crawler access as a ranking guarantee.", "Do not state that llms.txt affects Google ranking.", "Do not describe the current allowance of training crawlers as a deliberate policy — it is an unreviewed default.", "If robots.txt changes, update this file, llms.txt references, and source-governance maintenance notes together." ], "changes_2026_08_21": [ "Re-verified the OpenAI, Anthropic and Perplexity crawler tokens against their current official documentation. All nine tokens (OAI-SearchBot, GPTBot, OAI-AdsBot, ChatGPT-User, ClaudeBot, Claude-User, Claude-SearchBot, PerplexityBot, Perplexity-User) still have explicit Allow groups in robots.txt.", "Anthropic's current crawler page lists exactly three tokens: ClaudeBot, Claude-User, Claude-SearchBot. 'Claude-Code' is NOT among them, and neither is the legacy 'anthropic-ai'. The Claude-Code group in robots.txt is therefore a no-op — harmless, because the default group already allows everything, but it is not a documented crawler token and should not be cited as one.", "Context for why robots.txt matters more than llms.txt: a 12-week server-log study across 83 sites (2026-04-27 to 2026-07-19) measured GPTBot fetching robots.txt 3,990 times vs llms.txt 7 times, ClaudeBot 3,120 vs 9, and PerplexityBot 775 vs 0. Access is decided in robots.txt. The llms.txt catalogs are retained as low-cost, standards-track surfaces for agent clients, not as the primary route to citation." ] }