{"dataset_id":"citationgraph-dataset-crawler-visibility-taxonomy-2026-11","version":"2026.11.v1","title":"CitationGraph AI Crawler Role Taxonomy, Attribution Evidence Layers & Public Static GEO Weights","license":"https://creativecommons.org/licenses/by/4.0/","is_accessible_for_free":true,"canonical_url":"https://citationgraph.ai/en/data/ai-crawler-and-visibility-taxonomy","publisher":{"name":"Global Gravity Technology","entity_id":"https://www.global-gravity.com/#organization","product_entity_id":"https://citationgraph.ai/#software-application"},"scope_boundary_notice":"This open reference dataset documents CitationGraph public classification standards and static preview weights. It contains zero private customer telemetry and does not represent the complete commercial hosted dashboard.","ai_crawler_taxonomy":[{"bot_token":"GPTBot","operator":"OpenAI","primary_role":"model_training","robots_token":"GPTBot","signal_layer":"server_crawler_discovery","verification_method":"Published IP CIDR ranges + User-Agent"},{"bot_token":"OAI-SearchBot","operator":"OpenAI","primary_role":"search_index","robots_token":"OAI-SearchBot","signal_layer":"server_crawler_discovery","verification_method":"Published IP CIDR ranges + User-Agent"},{"bot_token":"ChatGPT-User","operator":"OpenAI","primary_role":"live_user_retrieval","robots_token":"ChatGPT-User","signal_layer":"live_retrieval_fetch","verification_method":"Published IP CIDR ranges + User-Agent"},{"bot_token":"ClaudeBot","operator":"Anthropic","primary_role":"model_training","robots_token":"ClaudeBot","signal_layer":"server_crawler_discovery","verification_method":"User-Agent + ASN attribution"},{"bot_token":"Claude-SearchBot","operator":"Anthropic","primary_role":"search_index","robots_token":"Claude-SearchBot","signal_layer":"server_crawler_discovery","verification_method":"User-Agent + ASN attribution"},{"bot_token":"Claude-User","operator":"Anthropic","primary_role":"live_user_retrieval","robots_token":"Claude-User","signal_layer":"live_retrieval_fetch","verification_method":"User-Agent + request header pattern"},{"bot_token":"PerplexityBot","operator":"Perplexity AI","primary_role":"search_index","robots_token":"PerplexityBot","signal_layer":"server_crawler_discovery","verification_method":"Published IP JSON list + User-Agent"},{"bot_token":"Perplexity-User","operator":"Perplexity AI","primary_role":"live_user_retrieval","robots_token":"Perplexity-User","signal_layer":"live_retrieval_fetch","verification_method":"Published IP JSON list + User-Agent"},{"bot_token":"Google-Extended","operator":"Google","primary_role":"model_training","robots_token":"Google-Extended","signal_layer":"server_crawler_discovery","verification_method":"Reverse/Forward DNS googlebot.com + policy token"},{"bot_token":"GoogleOther","operator":"Google","primary_role":"search_index","robots_token":"GoogleOther","signal_layer":"server_crawler_discovery","verification_method":"Reverse/Forward DNS googlebot.com"},{"bot_token":"Applebot-Extended","operator":"Apple","primary_role":"model_training","robots_token":"Applebot-Extended","signal_layer":"server_crawler_discovery","verification_method":"Reverse/Forward DNS applebot.apple.com + policy token"},{"bot_token":"Amazonbot","operator":"Amazon","primary_role":"search_index","robots_token":"Amazonbot","signal_layer":"server_crawler_discovery","verification_method":"Reverse/Forward DNS crawl.amazonbot.amazon"},{"bot_token":"Meta-ExternalAgent","operator":"Meta","primary_role":"model_training","robots_token":"Meta-ExternalAgent","signal_layer":"server_crawler_discovery","verification_method":"User-Agent + Meta ASN verification"},{"bot_token":"CCBot","operator":"Common Crawl","primary_role":"model_training","robots_token":"CCBot","signal_layer":"server_crawler_discovery","verification_method":"User-Agent + Common Crawl AWS egress metadata"}],"attribution_evidence_layers":[{"layer_id":"L1_server_crawler_truth","layer_name":"Server Crawler & Retrieval Telemetry","signal_source":"Edge/runtime HTTP request headers, status codes, and verified bot identity","deterministic_scope":"Proves crawler discovery and live answer-engine page fetches without client JS"},{"layer_id":"L2_citation_surface_readiness","layer_name":"Citation Surface & Entity Graph Readiness","signal_source":"Canonical HTML, JSON-LD @id graph, llms.txt, and .md dual-stack mirrors","deterministic_scope":"Evaluates structural eligibility for verifiable citations; does not guarantee citation"},{"layer_id":"L3_verified_ai_referral_session","layer_name":"First-Party AI Referral Session Attribution","signal_source":"Document referrer, UTM/citation parameters, and session landing page","deterministic_scope":"Separates human clicks arriving from AI answer engines from generic organic/direct visits"},{"layer_id":"L4_connected_outcome_context","layer_name":"Connected Search & Conversion Context","signal_source":"Authorized GSC query matrices, waitlist/lead events, and commerce outcomes","deterministic_scope":"Available in connected workspaces only; never inferred from public static previews"}],"public_geo_static_weights":[{"dimension_id":"infrastructure","dimension_weight_pct":25,"evaluated_signals":"HTTPS enforcement, HTTP 200 status, robots.txt crawl rules, XML sitemap presence","boundary_note":"Public static scan dimension; excludes private server logs"},{"dimension_id":"metadata","dimension_weight_pct":25,"evaluated_signals":"Title & meta description boundaries, self-referencing canonical, hreflang reciprocity, Open Graph","boundary_note":"Public static scan dimension; checks single-URL HTML head tags"},{"dimension_id":"structured_data","dimension_weight_pct":25,"evaluated_signals":"Valid JSON-LD syntax, Organization/SoftwareApplication/Dataset @id entities, FAQ/Article schema","boundary_note":"Public static scan dimension; validates machine-parsable entity definitions"},{"dimension_id":"ai_guidance","dimension_weight_pct":25,"evaluated_signals":"llms.txt / llms-full.txt availability, Markdown alternate Link headers, clean section headings","boundary_note":"Public static scan dimension; does not represent hosted multi-engine citation tracking"}]}