"id","domain","site_name","bucket","sector","hq_country","region","fetch_date_utc","robots_url","http_status","fetch_status","robots_size_bytes","robots_validity","has_ai_specific_rules","bot_GPTBot","bot_ClaudeBot","bot_CCBot","bot_Google-Extended","bot_Meta-ExternalAgent","bot_Applebot-Extended","bot_Bytespider","bot_Amazonbot","bot_Diffbot","bot_ImagesiftBot","bot_AI2Bot","bot_cohere-ai","bot_OAI-SearchBot","bot_Claude-SearchBot","bot_PerplexityBot","bot_Applebot","bot_YouBot","bot_DuckAssistBot","bot_ChatGPT-User","bot_Claude-User","bot_Perplexity-User","bot_MistralAI-User","blocks_all_training_bots","blocks_any_search_bot","blocks_any_core_search_bot","blocks_all_ai","llms_txt_status","llms_txt_shape","llms_txt_links","llms_txt_substance","llms_txt_absolute_url_occurrences","llms_txt_has_absolute_url","notes","robots_fetch_outcome","policy_determined","core_search_root_block","llms_classification","release_version","edition","robots_evidence_file","robots_archive_sha256","robots_indexed_sha256","robots_hash_verification","robots_requested_url","robots_final_url","llms_evidence_file","llms_archive_sha256","llms_indexed_sha256","llms_hash_verification","llms_requested_url","llms_final_url","llms_http_status","llms_content_type","llms_body_truncated","evidence_note","baseline_fetch_status","baseline_robots_fetch_outcome","baseline_policy_determined","baseline_http_status","baseline_fetch_date_utc","baseline_llms_txt_status","baseline_llms_classification","baseline_robots_evidence_file","baseline_llms_evidence_file","baseline_robots_archive_sha256","baseline_llms_archive_sha256","baseline_bot_GPTBot","baseline_bot_ClaudeBot","baseline_bot_CCBot","baseline_bot_Google-Extended","baseline_bot_Meta-ExternalAgent","baseline_bot_Applebot-Extended","baseline_bot_Bytespider","baseline_bot_Amazonbot","baseline_bot_Diffbot","baseline_bot_ImagesiftBot","baseline_bot_AI2Bot","baseline_bot_cohere-ai","baseline_bot_OAI-SearchBot","baseline_bot_Claude-SearchBot","baseline_bot_PerplexityBot","baseline_bot_Applebot","baseline_bot_YouBot","baseline_bot_DuckAssistBot","baseline_bot_ChatGPT-User","baseline_bot_Claude-User","baseline_bot_Perplexity-User","baseline_bot_MistralAI-User","result_view","robots_result_source","robots_verification_outcome","robots_verification_http","robots_verification_task_id","robots_verification_fetch_time","robots_verification_reason","robots_verification_response_file","robots_verification_attempt_count","llms_result_source","llms_verification_outcome","llms_verification_http","llms_verification_task_id","llms_verification_fetch_time","llms_verification_reason","llms_verification_response_file","llms_verification_attempt_count"
"1","nytimes.com","The New York Times","Global news & media","Newspaper","United States","Global","2026-10-21","https://www.nytimes.com/robots.txt","200","ok","8547","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nytimes.com.txt","04dee26f145ee77839c65ff45a4e1b887ae06be06b3b91fbc5043dc37277fcea","04dee26f145ee77839c65ff45a4e1b887ae06be06b3b91fbc5043dc37277fcea","Matches source index","https://nytimes.com/robots.txt","https://www.nytimes.com/robots.txt","","","e9e5717f2113fdd93ab6545310cac13501e00127141790c5d718686c46d166fe","No archived body","https://nytimes.com/llms.txt","https://www.nytimes.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nytimes.com.txt","","04dee26f145ee77839c65ff45a4e1b887ae06be06b3b91fbc5043dc37277fcea","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"2","washingtonpost.com","The Washington Post","Global news & media","Newspaper","United States","Global","2026-10-21","https://www.washingtonpost.com/robots.txt","200","ok","5804","valid_txt_200","True","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","True","True","False","unverified","","","","","","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Unverified","v3.2-verification","November 2026","raw/washingtonpost.com.txt","918970c4ae59aa1f1f31577e14eec569caf79d69660bd06c381516e4d5c49426","918970c4ae59aa1f1f31577e14eec569caf79d69660bd06c381516e4d5c49426","Matches source index","https://washingtonpost.com/robots.txt","https://www.washingtonpost.com/robots.txt","","","","No archived body","https://washingtonpost.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/washingtonpost.com.txt","","918970c4ae59aa1f1f31577e14eec569caf79d69660bd06c381516e4d5c49426","","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","","10222347-2533-0275-0000-fb14833cb43f","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"3","wsj.com","The Wall Street Journal","Global news & media","Newspaper","United States","Global","2026-10-21","https://www.wsj.com/robots.txt","200","ok","4298","valid_txt_200","True","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","False","True","True","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Unverified","v3.2-verification","November 2026","raw/wsj.com.txt","89210f23a6252c1f03f3615e35e81ab50714fcea36cb56e8ef7725fdb91de3aa","89210f23a6252c1f03f3615e35e81ab50714fcea36cb56e8ef7725fdb91de3aa","Matches source index","https://wsj.com/robots.txt","https://www.wsj.com/robots.txt","","","5129b0463087899afc1921fa7277db2b787ff93014190853ee7ff51707aa3768","No archived body","https://wsj.com/llms.txt","https://www.wsj.com/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/wsj.com.txt","","89210f23a6252c1f03f3615e35e81ab50714fcea36cb56e8ef7725fdb91de3aa","","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-08707e7fd137","2026-10-22 23:53:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"4","usatoday.com","USA Today","Global news & media","Newspaper","United States","Global","2026-10-21","https://www.usatoday.com/robots.txt","200","ok","16350","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","True","absent","","","","","","Baseline note:  llms.txt: HTTP 402 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/usatoday.com.txt","7ac5cfceb70ab70e88bc5023e98ea9e25efa1b553481003be1d1c9ed3c465186","7ac5cfceb70ab70e88bc5023e98ea9e25efa1b553481003be1d1c9ed3c465186","Matches source index","https://usatoday.com/robots.txt","https://www.usatoday.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://usatoday.com/llms.txt","https://www.usatoday.com/llms.txt/","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/usatoday.com.txt","","7ac5cfceb70ab70e88bc5023e98ea9e25efa1b553481003be1d1c9ed3c465186","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222353-2533-0275-0000-f269a5946cbf","2026-10-22 23:53:55 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"5","latimes.com","Los Angeles Times","Global news & media","Newspaper","United States","Global","2026-10-21","https://www.latimes.com:443/robots.txt","200","ok","1768","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host www.latimes.com:443","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/latimes.com.txt","6dc754f82f55902704f5e1cdd1f406e0f91246ce2238c92e9634f3335f57aab7","6dc754f82f55902704f5e1cdd1f406e0f91246ce2238c92e9634f3335f57aab7","Matches source index","https://latimes.com/robots.txt","https://www.latimes.com:443/robots.txt","","","7b44d2e7457137ed613e5916b2629369cdbdcc63736602b5b177fdeb30793c5e","No archived body","https://www.latimes.com/llms.txt","https://www.latimes.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/latimes.com.txt","","6dc754f82f55902704f5e1cdd1f406e0f91246ce2238c92e9634f3335f57aab7","","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"6","npr.org","NPR","Global news & media","Public broadcaster","United States","Global","2026-10-21","https://www.npr.org/robots.txt","200","ok","4736","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Allowed","Partial","Partial","Blocked","Blocked","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/npr.org.txt","53f9d15a3e089e2df2e24dc31beef07de952703ea14defd9f2e96afea4edc651","53f9d15a3e089e2df2e24dc31beef07de952703ea14defd9f2e96afea4edc651","Matches source index","https://npr.org/robots.txt","https://www.npr.org/robots.txt","","","26337b44d14c0ffeb89cc9d237f8763cedbb87a801051c9f84185603938c5423","No archived body","https://npr.org/llms.txt","https://www.npr.org/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/npr.org.txt","","53f9d15a3e089e2df2e24dc31beef07de952703ea14defd9f2e96afea4edc651","","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Allowed","Partial","Partial","Blocked","Blocked","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"7","cnn.com","CNN","Global news & media","Broadcaster","United States","Global","2026-10-21","https://www.cnn.com/robots.txt","200","ok","3671","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cnn.com.txt","2f94c14d6285e51945fe4c9e6999d47425a421a4ab9aae347ded29e6e5b05144","2f94c14d6285e51945fe4c9e6999d47425a421a4ab9aae347ded29e6e5b05144","Matches source index","https://cnn.com/robots.txt","https://www.cnn.com/robots.txt","","","d269e6896f6eafd73b6cbf4c846c843d07f6b931f49f1dbfa61cb4da20fe6e89","No archived body","https://cnn.com/llms.txt","https://edition.cnn.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cnn.com.txt","","2f94c14d6285e51945fe4c9e6999d47425a421a4ab9aae347ded29e6e5b05144","","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"8","nbcnews.com","NBC News","Global news & media","Broadcaster","United States","Global","2026-10-21","https://www.nbcnews.com/robots.txt","200","ok","14591","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nbcnews.com.txt","9fc48b1225c428fc640b3e49fb9e62a65fb9795e6bcf75d15856b7044a3ebe10","9fc48b1225c428fc640b3e49fb9e62a65fb9795e6bcf75d15856b7044a3ebe10","Matches source index","https://nbcnews.com/robots.txt","https://www.nbcnews.com/robots.txt","","","0ef6158ef36f5790456ba79d9b2b562412699b5560901b2f2dddf8d069b528e7","No archived body","https://nbcnews.com/llms.txt","https://www.nbcnews.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nbcnews.com.txt","","9fc48b1225c428fc640b3e49fb9e62a65fb9795e6bcf75d15856b7044a3ebe10","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"9","foxnews.com","Fox News","Global news & media","Broadcaster","United States","Global","2026-10-21","https://www.foxnews.com/robots.txt","200","ok","301","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","21","has_strict_links","21","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/foxnews.com.txt","4abc1bd3b4d0b87c64de7eba0eafe63dd90415f25d12133d88d7225cacb9dc07","4abc1bd3b4d0b87c64de7eba0eafe63dd90415f25d12133d88d7225cacb9dc07","Matches source index","https://foxnews.com/robots.txt","https://www.foxnews.com/robots.txt","llms/foxnews.com.txt","4a078f7b2ef22b42dd54845ca31b4eb1782e096c3eab0e4ada5733ad49c92327","4a078f7b2ef22b42dd54845ca31b4eb1782e096c3eab0e4ada5733ad49c92327","Matches source index","https://foxnews.com/llms.txt","https://www.foxnews.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/foxnews.com.txt","llms/foxnews.com.txt","4abc1bd3b4d0b87c64de7eba0eafe63dd90415f25d12133d88d7225cacb9dc07","4a078f7b2ef22b42dd54845ca31b4eb1782e096c3eab0e4ada5733ad49c92327","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"10","cbsnews.com","CBS News","Global news & media","Broadcaster","United States","Global","2026-10-21","https://www.cbsnews.com/robots.txt","200","ok","950","valid_txt_200","True","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cbsnews.com.txt","3183f29f832838e64ddcdfe0fa253efe5003d650c636ba40efa6cfcfe4489570","3183f29f832838e64ddcdfe0fa253efe5003d650c636ba40efa6cfcfe4489570","Matches source index","https://cbsnews.com/robots.txt","https://www.cbsnews.com/robots.txt","","","3d88bd39b39111fd2fa30ff8a2b60c09e63459c686535a245b235be23dca0b4f","No archived body","https://cbsnews.com/llms.txt","https://www.cbsnews.com/llms.txt/","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cbsnews.com.txt","","3183f29f832838e64ddcdfe0fa253efe5003d650c636ba40efa6cfcfe4489570","","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"11","bloomberg.com","Bloomberg","Global news & media","Financial news","United States","Global","2026-10-21","https://www.bloomberg.com/robots.txt","200","ok","10640","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","False","True","True","False","present","h1_markdown","130","has_strict_links","130","True","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/bloomberg.com.txt","6879b4e3afbc243a8422c6e1331cad0f7c85a7817612bf65a56d27cd6bc9ebf7","6879b4e3afbc243a8422c6e1331cad0f7c85a7817612bf65a56d27cd6bc9ebf7","Matches source index","https://bloomberg.com/robots.txt","https://www.bloomberg.com/robots.txt","crawler-api-verification-pass/raw/llms/bloomberg.com--10222341-2533-0275-0000-d9a3cc327e44.txt","fc6a8eff9504484e1a8b7cae51c08e39bf922af5dc76aaa99fa92e3b6bec1f08","","API-returned UTF-8 body; byte length matched provider size","https://bloomberg.com/llms.txt","https://www.bloomberg.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/bloomberg.com.txt","","6879b4e3afbc243a8422c6e1331cad0f7c85a7817612bf65a56d27cd6bc9ebf7","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222341-2533-0275-0000-d9a3cc327e44","2026-10-22 23:41:04 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-01.json","1"
"12","forbes.com","Forbes","Global news & media","Business news","United States","Global","2026-10-21","https://www.forbes.com/robots.txt","200","ok","33649","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Allowed","Allowed","Blocked","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/forbes.com.txt","29e98ab89e88236d171f529a26cb95984f1021926f62c26e9a0656cebf364b5d","29e98ab89e88236d171f529a26cb95984f1021926f62c26e9a0656cebf364b5d","Matches source index","https://forbes.com/robots.txt","https://www.forbes.com/robots.txt","","","17c550f26e692b07cc41d0d6532c2b369bfe3bade1126b1ba2b8bc9655d0d374","No archived body","https://forbes.com/llms.txt","https://www.forbes.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/forbes.com.txt","","29e98ab89e88236d171f529a26cb95984f1021926f62c26e9a0656cebf364b5d","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Allowed","Allowed","Blocked","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"13","time.com","TIME","Global news & media","Magazine","United States","Global","2026-10-21","https://time.com/robots.txt","200","ok","2190","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","1","True","Baseline note:  llms.txt: HTTP 406 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/time.com.txt","9ae14e98633c12d81afbf8f651cbff2b3d3b7c59e2d6851b329107c7e7bd5dc8","9ae14e98633c12d81afbf8f651cbff2b3d3b7c59e2d6851b329107c7e7bd5dc8","Matches source index","https://time.com/robots.txt","https://time.com/robots.txt","crawler-api-verification-pass/raw/llms/time.com--10222341-2533-0275-0000-59e2529b293f.txt","33b2b869b9498b20f104002731adc96644345ce5f8d4cd4823c4e9f54499231f","","API-returned UTF-8 body; byte length matched provider size","https://time.com/llms.txt","https://time.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/time.com.txt","","9ae14e98633c12d81afbf8f651cbff2b3d3b7c59e2d6851b329107c7e7bd5dc8","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222341-2533-0275-0000-59e2529b293f","2026-10-22 23:41:02 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-01.json","1"
"14","theatlantic.com","The Atlantic","Global news & media","Magazine","United States","Global","2026-10-21","https://www.theatlantic.com/robots.txt","200","ok","3722","valid_txt_200","True","Allowed","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Allowed","Blocked","Blocked","Allowed","Partial","Allowed","Allowed","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/theatlantic.com.txt","500c1ee9d0c39ec3890bc0d1433e8506008290f278ce37daf6eed7358ef1bd5a","500c1ee9d0c39ec3890bc0d1433e8506008290f278ce37daf6eed7358ef1bd5a","Matches source index","https://theatlantic.com/robots.txt","https://www.theatlantic.com/robots.txt","","","4cb970eee098ba72307faf88ccf4476f2a438cc4eeab865683fc135146921b2a","No archived body","https://theatlantic.com/llms.txt","https://www.theatlantic.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/theatlantic.com.txt","","500c1ee9d0c39ec3890bc0d1433e8506008290f278ce37daf6eed7358ef1bd5a","","Allowed","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Allowed","Blocked","Blocked","Allowed","Partial","Allowed","Allowed","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"15","politico.com","Politico","Global news & media","Digital news","United States","Global","2026-10-21","https://www.politico.com:443/robots.txt","200","ok","5786","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","Baseline note: robots.txt redirects to host www.politico.com:443 politico.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/politico.com.txt","60d23f61bee9b1e6c11b21b17265e3fdc667e06a14e4e2907a222a4c02d323b6","60d23f61bee9b1e6c11b21b17265e3fdc667e06a14e4e2907a222a4c02d323b6","Matches source index","https://politico.com/robots.txt","https://www.politico.com:443/robots.txt","","","","No stored body; provider HTTP response in task record","https://politico.com/llms.txt","https://www.politico.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/politico.com.txt","","60d23f61bee9b1e6c11b21b17265e3fdc667e06a14e4e2907a222a4c02d323b6","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222341-2533-0275-0000-5da4128a36c0","2026-10-22 23:41:02 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-01.json","1"
"16","axios.com","Axios","Global news & media","Digital news","United States","Global","2026-10-21","https://www.axios.com/robots.txt","200","ok","645","valid_txt_200","True","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/axios.com.txt","9dbf5ea2259b2cb8a0236f5208c2bab2e0e83985be508732d648ca27fd32f9b5","9dbf5ea2259b2cb8a0236f5208c2bab2e0e83985be508732d648ca27fd32f9b5","Matches source index","https://axios.com/robots.txt","https://www.axios.com/robots.txt","","","ed6d7a8e2a6dd529710fe05bf6a63142b14300fe64a6a1932b4b86481c2fb146","No archived body","https://axios.com/llms.txt","https://www.axios.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/axios.com.txt","","9dbf5ea2259b2cb8a0236f5208c2bab2e0e83985be508732d648ca27fd32f9b5","","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"17","huffpost.com","HuffPost","Global news & media","Digital news","United States","Global","2026-10-21","https://www.huffpost.com/robots.txt","200","ok","2562","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","False","True","True","False","absent","","","","","","Baseline note:  llms.txt: HTTP 406 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/huffpost.com.txt","a7c45b9f6a0cb53b1aa7459065ec1306fa931db18eae15c4cdbac85486911e58","a7c45b9f6a0cb53b1aa7459065ec1306fa931db18eae15c4cdbac85486911e58","Matches source index","https://huffpost.com/robots.txt","https://www.huffpost.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://huffpost.com/llms.txt","https://www.huffpost.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/huffpost.com.txt","","a7c45b9f6a0cb53b1aa7459065ec1306fa931db18eae15c4cdbac85486911e58","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222341-2533-0275-0000-75ef38c4c9c9","2026-10-22 23:41:01 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-01.json","1"
"18","businessinsider.com","Business Insider","Global news & media","Digital news","United States","Global","2026-10-21","https://www.businessinsider.com/robots.txt","200","ok","2231","valid_txt_200","True","Partial","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","9","has_strict_links","9","True","businessinsider.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/businessinsider.com.txt","89f51cf7e3b532fe80b2057cf13765fdc52bc55aafc263e876a37fe120873ddd","89f51cf7e3b532fe80b2057cf13765fdc52bc55aafc263e876a37fe120873ddd","Matches source index","https://businessinsider.com/robots.txt","https://www.businessinsider.com/robots.txt","llms/businessinsider.com.txt","2dc562cd1793905965b6f811da015e20f9b9740c9f4fca395eee0c394b366304","2dc562cd1793905965b6f811da015e20f9b9740c9f4fca395eee0c394b366304","Matches source index","https://businessinsider.com/llms.txt","https://www.businessinsider.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/businessinsider.com.txt","llms/businessinsider.com.txt","89f51cf7e3b532fe80b2057cf13765fdc52bc55aafc263e876a37fe120873ddd","2dc562cd1793905965b6f811da015e20f9b9740c9f4fca395eee0c394b366304","Partial","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"19","bbc.com","BBC","Global news & media","Public broadcaster","United Kingdom","Global","2026-10-21","https://www.bbc.com/robots.txt","200","ok","6169","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bbc.com.txt","37a8567b049d9a34ca7b7dcb420ed3b8cbd929f1720a1c57c03ad28a1efec0b7","37a8567b049d9a34ca7b7dcb420ed3b8cbd929f1720a1c57c03ad28a1efec0b7","Matches source index","https://bbc.com/robots.txt","https://www.bbc.com/robots.txt","","","1d76e54236d324a5855ecc22ec9b25267ddcef008983b3e4773bc89172e5d24f","No archived body","https://bbc.com/llms.txt","https://www.bbc.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bbc.com.txt","","37a8567b049d9a34ca7b7dcb420ed3b8cbd929f1720a1c57c03ad28a1efec0b7","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"20","theguardian.com","The Guardian","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.theguardian.com/robots.txt","200","ok","2873","valid_txt_200","True","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","False","True","True","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/theguardian.com.txt","3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2","3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2","Matches source index","https://theguardian.com/robots.txt","https://www.theguardian.com/robots.txt","","","5a504aff032fe8faab92965fc40e0c4bcaa7709d90347f0d14415572e6a0586a","No archived body","https://theguardian.com/llms.txt","https://www.theguardian.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/theguardian.com.txt","","3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2","","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"21","telegraph.co.uk","The Telegraph","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.telegraph.co.uk/robots.txt","200","ok","914","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","True","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/telegraph.co.uk.txt","78276308acadd26b924db3550cee612266641189a0568b8c1f87f89388923101","78276308acadd26b924db3550cee612266641189a0568b8c1f87f89388923101","Matches source index","https://telegraph.co.uk/robots.txt","https://www.telegraph.co.uk/robots.txt","","","8ef21774d7cb46997a7aae8a13f400d8726367e13af02ff93ca5b4e6cd4a5dc4","No archived body","https://telegraph.co.uk/llms.txt","https://www.telegraph.co.uk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/telegraph.co.uk.txt","","78276308acadd26b924db3550cee612266641189a0568b8c1f87f89388923101","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"22","thetimes.co.uk","The Times","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.thetimes.co.uk/robots.txt","200","ok","1700","valid_txt_200","True","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/thetimes.co.uk.txt","9d44c43a9e47fb7a2fe708fd37d36b1cc4cacee5af51f7ceb592c0aa3387b4d2","9d44c43a9e47fb7a2fe708fd37d36b1cc4cacee5af51f7ceb592c0aa3387b4d2","Matches source index","https://thetimes.co.uk/robots.txt","https://www.thetimes.co.uk/robots.txt","llms/thetimes.co.uk.txt","0c1392d14fef0856ae5ef029bac991157ea487148e85d5f47a6e04d9e100a217","0c1392d14fef0856ae5ef029bac991157ea487148e85d5f47a6e04d9e100a217","Matches source index","https://thetimes.co.uk/llms.txt","https://www.thetimes.com/llms.txt","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/thetimes.co.uk.txt","llms/thetimes.co.uk.txt","9d44c43a9e47fb7a2fe708fd37d36b1cc4cacee5af51f7ceb592c0aa3387b4d2","0c1392d14fef0856ae5ef029bac991157ea487148e85d5f47a6e04d9e100a217","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"23","independent.co.uk","The Independent","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.independent.co.uk/robots.txt","200","ok","1305","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/independent.co.uk.txt","b45123ef0320e9ca197046d8c217d77563f194a4d92c3c2e887c0d04198bafa5","b45123ef0320e9ca197046d8c217d77563f194a4d92c3c2e887c0d04198bafa5","Matches source index","https://independent.co.uk/robots.txt","https://www.independent.co.uk/robots.txt","","","89a3dac41ca84292d18258f410153affdfe4ef04839478ed57e4e709e8caddd0","No archived body","https://independent.co.uk/llms.txt","https://independent.co.uk/llms.txt","410","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/independent.co.uk.txt","","b45123ef0320e9ca197046d8c217d77563f194a4d92c3c2e887c0d04198bafa5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"24","ft.com","Financial Times","Global news & media","Financial news","United Kingdom","Global","2026-10-21","https://www.ft.com/robots.txt","200","ok","1556","valid_txt_200","True","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","False","True","True","False","absent","","","","","","ft.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ft.com.txt","57ce1e58ae57b17699c48037deda0f549296f5c0bfe44e92cf7af6deb6767b4b","57ce1e58ae57b17699c48037deda0f549296f5c0bfe44e92cf7af6deb6767b4b","Matches source index","https://ft.com/robots.txt","https://www.ft.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://ft.com/llms.txt","https://www.ft.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ft.com.txt","","57ce1e58ae57b17699c48037deda0f549296f5c0bfe44e92cf7af6deb6767b4b","","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"25","economist.com","The Economist","Global news & media","Magazine","United Kingdom","Global","2026-10-21","https://www.economist.com/robots.txt","200","ok","2664","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/economist.com.txt","5b0aecabb2641f39b0f0f6f3f4ab695575f1454b051cf4c9ea43b038f40130ce","5b0aecabb2641f39b0f0f6f3f4ab695575f1454b051cf4c9ea43b038f40130ce","Matches source index","https://economist.com/robots.txt","https://www.economist.com/robots.txt","","","9b7e897e9321e0d5db0b84a765bffa312d666d9cb34fae97eaf2580587041b6e","No archived body","https://economist.com/llms.txt","https://www.economist.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/economist.com.txt","","5b0aecabb2641f39b0f0f6f3f4ab695575f1454b051cf4c9ea43b038f40130ce","","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"26","dailymail.co.uk","Daily Mail","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.dailymail.com/robots.txt","200","ok","5452","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","present","h1_markdown","164","has_strict_links","164","True","robots.txt redirects to host www.dailymail.com dailymail.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/dailymail.co.uk.txt","4d3c6789fa319d2acd0abacd0a9bed5fa67733bae6bb08808ddb53ff358a22a6","4d3c6789fa319d2acd0abacd0a9bed5fa67733bae6bb08808ddb53ff358a22a6","Matches source index","https://dailymail.co.uk/robots.txt","https://www.dailymail.com/robots.txt","llms/dailymail.co.uk.txt","9e321fdb7dc727019df95fa5bfc7cba7b2e03789b693360bcce019e3360e8be0","9e321fdb7dc727019df95fa5bfc7cba7b2e03789b693360bcce019e3360e8be0","Matches source index","https://dailymail.co.uk/llms.txt","https://www.dailymail.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/dailymail.co.uk.txt","llms/dailymail.co.uk.txt","4d3c6789fa319d2acd0abacd0a9bed5fa67733bae6bb08808ddb53ff358a22a6","9e321fdb7dc727019df95fa5bfc7cba7b2e03789b693360bcce019e3360e8be0","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"27","mirror.co.uk","Daily Mirror","Global news & media","Newspaper","United Kingdom","Global","2026-10-21","https://www.mirror.co.uk/robots.txt","200","ok","1900","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mirror.co.uk.txt","f278e4234da789b8de6a5eb78a8c76cb28613759e488c291d254ff76d27719e8","f278e4234da789b8de6a5eb78a8c76cb28613759e488c291d254ff76d27719e8","Matches source index","https://mirror.co.uk/robots.txt","https://www.mirror.co.uk/robots.txt","","","15fcff7675eacd6cfbbf009b94f1e8e977f4964a5c816d93dff0df2aefd4b36c","No archived body","https://mirror.co.uk/llms.txt","https://www.mirror.co.uk/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mirror.co.uk.txt","","f278e4234da789b8de6a5eb78a8c76cb28613759e488c291d254ff76d27719e8","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"28","itv.com","ITV","Global news & media","Broadcaster","United Kingdom","Global","2026-10-22","https://itv.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.itv.com', port=80): Read timed out. (read timeout=20) itv.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://itv.com/robots.txt","https://itv.com/robots.txt","","","","No archived body","https://itv.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222347-2533-0275-0000-5f8b10682457","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-pilot.json","2","Frozen v3.1.2 baseline","Undetermined","","10222353-2533-0275-0000-a2657bfff565","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"29","reuters.com","Reuters","Global news & media","Wire service","","Global","2026-10-21","https://www.reuters.com/robots.txt","200","ok","4627","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","True","True","True","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 401 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Unverified","v3.2-verification","November 2026","raw/reuters.com.txt","a6610234e4c1c34b9ea02a5e3af72dd4d219e2f3cd97ba6b1a7fe37166fa64a7","a6610234e4c1c34b9ea02a5e3af72dd4d219e2f3cd97ba6b1a7fe37166fa64a7","Matches source index","https://reuters.com/robots.txt","https://www.reuters.com/robots.txt","","","aff7ac72dab8427210b7490e81045222255d3aa1b281cbef435e3423c1d4a714","No archived body","https://reuters.com/llms.txt","https://www.reuters.com/llms.txt","401","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/reuters.com.txt","","a6610234e4c1c34b9ea02a5e3af72dd4d219e2f3cd97ba6b1a7fe37166fa64a7","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","401","10222353-2533-0275-0000-69ae65bbcc5d","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"30","apnews.com","Associated Press","Global news & media","Wire service","United States","Global","2026-10-21","https://apnews.com/robots.txt","200","ok","1326","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","False","True","True","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/apnews.com.txt","fad8a37c40bf85e344b17ae483d9d81e363a90eae8c4f53ea2978787e6ef63da","fad8a37c40bf85e344b17ae483d9d81e363a90eae8c4f53ea2978787e6ef63da","Matches source index","https://apnews.com/robots.txt","https://apnews.com/robots.txt","","","b65a320ab44d0761c998744b20aff5f366a28e2bc51492527bbf1230a70493f6","No archived body","https://apnews.com/llms.txt","https://apnews.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/apnews.com.txt","","fad8a37c40bf85e344b17ae483d9d81e363a90eae8c4f53ea2978787e6ef63da","","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"31","afp.com","AFP","Global news & media","Wire service","France","Global","2026-10-22","https://www.afp.com/robots.txt","200","ok","2027","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","0","False","com', port=443): Max retries exceeded with url: /com', port=443):","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/afp.com.txt","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","Matches source index","https://afp.com/robots.txt","https://www.afp.com/robots.txt","llms/afp.com.txt","5b96a1b305c46c3520e34a792ebbb3eeb4d58def9ea63bb36d46d3152273585d","5b96a1b305c46c3520e34a792ebbb3eeb4d58def9ea63bb36d46d3152273585d","Matches source index","https://www.afp.com/llms.txt","https://www.afp.com/llms.txt","200","text/markdown; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/afp.com.txt","llms/afp.com.txt","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","5b96a1b305c46c3520e34a792ebbb3eeb4d58def9ea63bb36d46d3152273585d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"32","lemonde.fr","Le Monde","Global news & media","Newspaper","France","Global","2026-10-21","https://www.lemonde.fr/robots.txt","200","ok","13226","valid_txt_200","True","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","False","True","True","False","absent","","","","","","Baseline note: fr', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 402 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/lemonde.fr.txt","843d2d9af6837d4c052a7d7b88c2a76463b4ac9880d191dbd8800b14c67c85ca","843d2d9af6837d4c052a7d7b88c2a76463b4ac9880d191dbd8800b14c67c85ca","Matches source index","https://lemonde.fr/robots.txt","https://www.lemonde.fr/robots.txt","","","","No stored body; provider HTTP response in task record","https://lemonde.fr/llms.txt","https://www.lemonde.fr/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/lemonde.fr.txt","","843d2d9af6837d4c052a7d7b88c2a76463b4ac9880d191dbd8800b14c67c85ca","","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222347-2533-0275-0000-d0848a608b7b","2026-10-22 23:47:23 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"33","france24.com","France 24","Global news & media","Broadcaster","France","Global","2026-10-21","https://www.france24.com/robots.txt","200","ok","12929","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","unverified","","","","","","Baseline note: france24.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Unverified","v3.2-verification","November 2026","raw/france24.com.txt","e9c6fffa8dd78b8d26225f1a6631e52b301b9b90bd415c91ace82c640b47dd2f","e9c6fffa8dd78b8d26225f1a6631e52b301b9b90bd415c91ace82c640b47dd2f","Matches source index","https://france24.com/robots.txt","https://www.france24.com/robots.txt","","","103554725d2871bdf92690aa92bf97f9a46197974306cd2b144fe83ccc4e5e0f","No archived body","https://france24.com/llms.txt","https://www.france24.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/france24.com.txt","","e9c6fffa8dd78b8d26225f1a6631e52b301b9b90bd415c91ace82c640b47dd2f","","Blocked","Blocked","Blocked","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222347-2533-0275-0000-887c3db43548","2026-10-22 23:47:22 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"34","euronews.com","Euronews","Global news & media","Broadcaster","France","Global","2026-10-21","https://www.euronews.com:443/robots.txt","200","ok","2387","valid_txt_200","True","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: robots.txt redirects to host www.euronews.com:443 llms.txt: HTTP 406 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/euronews.com.txt","533a27093f589541f67b56f56ec0e25c30e66368b3798acc01f98fe613a7c51e","533a27093f589541f67b56f56ec0e25c30e66368b3798acc01f98fe613a7c51e","Matches source index","https://euronews.com/robots.txt","https://www.euronews.com:443/robots.txt","","","","No stored body; provider HTTP response in task record","https://euronews.com/llms.txt","https://www.euronews.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/euronews.com.txt","","533a27093f589541f67b56f56ec0e25c30e66368b3798acc01f98fe613a7c51e","","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222341-2533-0275-0000-e4acdc9e2269","2026-10-22 23:41:01 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-01.json","1"
"35","spiegel.de","Der Spiegel","Global news & media","Magazine","Germany","Global","2026-10-21","https://www.spiegel.de/robots.txt","200","ok","6634","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Blocked","False","True","True","False","absent","","","","","","de', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/spiegel.de.txt","95f0a83e7d047a5a58ef261b8a7563234379d9c47a64748d58b383298f760d13","95f0a83e7d047a5a58ef261b8a7563234379d9c47a64748d58b383298f760d13","Matches source index","https://spiegel.de/robots.txt","https://www.spiegel.de/robots.txt","","","774ca0539ebd6013bbea2399e147841b7754109fb1f557674e10f25e499932db","No archived body","https://spiegel.de/llms.txt","https://www.spiegel.de/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/spiegel.de.txt","","95f0a83e7d047a5a58ef261b8a7563234379d9c47a64748d58b383298f760d13","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"36","dw.com","Deutsche Welle","Global news & media","Public broadcaster","Germany","Global","2026-10-21","https://www.dw.com/robots.txt","200","ok","2874","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/dw.com.txt","f7daa7ad2ba66d113657a45c8ef5269c60701e204bee0c0560f196b3b985c98c","f7daa7ad2ba66d113657a45c8ef5269c60701e204bee0c0560f196b3b985c98c","Matches source index","https://dw.com/robots.txt","https://www.dw.com/robots.txt","","","4f98df5ae53a2e69baf52b73b005bb62fc02cb5dfbf5899a3b1ad572b5a3c3a0","No archived body","https://dw.com/llms.txt","https://www.dw.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/dw.com.txt","","f7daa7ad2ba66d113657a45c8ef5269c60701e204bee0c0560f196b3b985c98c","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"37","bild.de","Bild","Global news & media","Newspaper","Germany","Global","2026-10-21","https://www.bild.de/robots.txt","200","ok","3053","valid_txt_200","True","Partial","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bild.de.txt","f2af535c9e54f1817009854d5e5ac2980d80fd910b2a5af4ea6aa45e3322d497","f2af535c9e54f1817009854d5e5ac2980d80fd910b2a5af4ea6aa45e3322d497","Matches source index","https://bild.de/robots.txt","https://www.bild.de/robots.txt","","","750f7d7cb8e35bb7a233ff8a7c1ed99160f6d90e45d9fd1df3beab246bc008d7","No archived body","https://bild.de/llms.txt","https://www.bild.de/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bild.de.txt","","f2af535c9e54f1817009854d5e5ac2980d80fd910b2a5af4ea6aa45e3322d497","","Partial","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"38","faz.net","Frankfurter Allgemeine Zeitung","Global news & media","Newspaper","Germany","Global","2026-10-21","https://www.faz.net/robots.txt","200","ok","3363","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/faz.net.txt","e0b697f51248cd0d2abfd135e343bd22d5e4ff396ca08b5ec64833c07bc4f93d","e0b697f51248cd0d2abfd135e343bd22d5e4ff396ca08b5ec64833c07bc4f93d","Matches source index","https://faz.net/robots.txt","https://www.faz.net/robots.txt","","","d293d9f3fbc0f1fb76f4a9332749d3ef8340d93d4eed11525d926d87a6ef484f","No archived body","https://faz.net/llms.txt","https://www.faz.net/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/faz.net.txt","","e0b697f51248cd0d2abfd135e343bd22d5e4ff396ca08b5ec64833c07bc4f93d","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"39","sueddeutsche.de","Sueddeutsche Zeitung","Global news & media","Newspaper","Germany","Global","2026-10-21","https://www.sueddeutsche.de/robots.txt","200","ok","2273","valid_txt_200","True","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/sueddeutsche.de.txt","591085a9b5f648b66068ac4c6a8bc8af7ba55d0b13b99bb73ca46a1debcbb4f3","591085a9b5f648b66068ac4c6a8bc8af7ba55d0b13b99bb73ca46a1debcbb4f3","Matches source index","https://sueddeutsche.de/robots.txt","https://www.sueddeutsche.de/robots.txt","llms/sueddeutsche.de.txt","6f832592ca0a2f2a822eeaf75f16115deb31d2682bc150c6c30d294ee371a759","6f832592ca0a2f2a822eeaf75f16115deb31d2682bc150c6c30d294ee371a759","Matches source index","https://sueddeutsche.de/llms.txt","https://www.sueddeutsche.de/","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/sueddeutsche.de.txt","llms/sueddeutsche.de.txt","591085a9b5f648b66068ac4c6a8bc8af7ba55d0b13b99bb73ca46a1debcbb4f3","6f832592ca0a2f2a822eeaf75f16115deb31d2682bc150c6c30d294ee371a759","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"40","elpais.com","El Pais","Global news & media","Newspaper","Spain","Global","2026-10-21","https://elpais.com/robots.txt","200","ok","7777","valid_txt_200","True","Partial","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/elpais.com.txt","7f5346154ef695787dde0a421fa22c41e47aff722e2f74f228979897c5392536","7f5346154ef695787dde0a421fa22c41e47aff722e2f74f228979897c5392536","Matches source index","https://elpais.com/robots.txt","https://elpais.com/robots.txt","","","1c250d82952286183d0d5b3d5a344516599085bbe01cd0a132e9fb22279d5dd5","No archived body","https://elpais.com/llms.txt","https://elpais.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/elpais.com.txt","","7f5346154ef695787dde0a421fa22c41e47aff722e2f74f228979897c5392536","","Partial","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"41","rtve.es","RTVE","Global news & media","Public broadcaster","Spain","Global","2026-10-21","https://www.rtve.es/akamai/robots.txt","200","ok","10319","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/rtve.es.txt","012fe60f6e2f03d3961ab3c468d739842d37befd5ef1c84c5a46f70f4d90d82a","012fe60f6e2f03d3961ab3c468d739842d37befd5ef1c84c5a46f70f4d90d82a","Matches source index","https://rtve.es/robots.txt","https://www.rtve.es/akamai/robots.txt","","","0610da0511e3857a8e4d1785e84c7e7b967bf0d44df3e0b6bfe6066c64934281","No archived body","https://rtve.es/llms.txt","https://www.rtve.es/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/rtve.es.txt","","012fe60f6e2f03d3961ab3c468d739842d37befd5ef1c84c5a46f70f4d90d82a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"42","corriere.it","Corriere della Sera","Global news & media","Newspaper","Italy","Global","2026-10-21","https://www.corriere.it/robots.txt","200","ok","10124","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","False","True","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/corriere.it.txt","51b3146315baf7dde0d32ca6b2806b9ee320a6198b66436929daa955562c27b2","51b3146315baf7dde0d32ca6b2806b9ee320a6198b66436929daa955562c27b2","Matches source index","https://corriere.it/robots.txt","https://www.corriere.it/robots.txt","","","4bcef9b580195e4b508f2c826bf28c20b9fd415d14b513a9731b5f308b9b3e6c","No archived body","https://corriere.it/llms.txt","https://www.corriere.it/llms.txt","404","text/html; charset=ISO-8859-1","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/corriere.it.txt","","51b3146315baf7dde0d32ca6b2806b9ee320a6198b66436929daa955562c27b2","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"43","repubblica.it","La Repubblica","Global news & media","Newspaper","Italy","Global","2026-10-21","https://www.repubblica.it/robots.txt","200","ok","7368","valid_txt_200","True","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Allowed","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","False","True","True","False","present","h1_markdown","38","has_strict_links","38","True","Baseline note: repubblica.it', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/repubblica.it.txt","2c064ff50d1ae310d617dbc6022174c29092012c3fa3f15d1f5f951c50189408","2c064ff50d1ae310d617dbc6022174c29092012c3fa3f15d1f5f951c50189408","Matches source index","https://repubblica.it/robots.txt","https://www.repubblica.it/robots.txt","crawler-api-verification-pass/raw/llms/repubblica.it--10222341-2533-0275-0000-5decb5c23b07.txt","6a7332964fdd8c4746d949a6cb64c62cd12611078999cbbd0a4eacb5d86d58ed","","API-returned UTF-8 body; byte length matched provider size","https://repubblica.it/llms.txt","https://www.repubblica.it/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/repubblica.it.txt","","2c064ff50d1ae310d617dbc6022174c29092012c3fa3f15d1f5f951c50189408","","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Allowed","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222341-2533-0275-0000-5decb5c23b07","2026-10-22 23:41:01 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-01.json","1"
"44","nos.nl","NOS","Global news & media","Public broadcaster","Netherlands","Global","2026-10-21","https://nos.nl/robots.txt","200","ok","1287","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nos.nl.txt","66758d918183dfb038d632775241420ede88c107f64dcf32cdf5650b82d83c75","66758d918183dfb038d632775241420ede88c107f64dcf32cdf5650b82d83c75","Matches source index","https://nos.nl/robots.txt","https://nos.nl/robots.txt","","","c60bbf7640292d3ae442a98ab371fa5e7ec2527f30158a6f0823ffa27c9dfeb8","No archived body","https://nos.nl/llms.txt","https://nos.nl/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nos.nl.txt","","66758d918183dfb038d632775241420ede88c107f64dcf32cdf5650b82d83c75","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"45","cbc.ca","CBC","Global news & media","Public broadcaster","Canada","Global","2026-10-21","https://www.cbc.ca/robots.txt","200","ok","2195","valid_txt_200","True","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cbc.ca.txt","e4a9683c27c03ce852ccd7ec5aea104171019e725943d4b2c69796a8092b54ae","e4a9683c27c03ce852ccd7ec5aea104171019e725943d4b2c69796a8092b54ae","Matches source index","https://cbc.ca/robots.txt","https://www.cbc.ca/robots.txt","","","d246d819f23f10bc8bb06298ac561444c2c647db3b6a17108e7d42db0d133c96","No archived body","https://cbc.ca/llms.txt","https://www.cbc.ca/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cbc.ca.txt","","e4a9683c27c03ce852ccd7ec5aea104171019e725943d4b2c69796a8092b54ae","","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"46","theglobeandmail.com","The Globe and Mail","Global news & media","Newspaper","Canada","Global","2026-10-21","https://www.theglobeandmail.com/robots.txt","200","ok","7529","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/theglobeandmail.com.txt","54dae88f29b7a33a8310de8ef4ffab54a370a601258e1f9a95b3bc0975ed9f77","54dae88f29b7a33a8310de8ef4ffab54a370a601258e1f9a95b3bc0975ed9f77","Matches source index","https://theglobeandmail.com/robots.txt","https://www.theglobeandmail.com/robots.txt","","","3386b106cd5f8839dc2289a50388e34bc6db712f8dd99a2afb9b8cdcc13e8d52","No archived body","https://theglobeandmail.com/llms.txt","https://www.theglobeandmail.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/theglobeandmail.com.txt","","54dae88f29b7a33a8310de8ef4ffab54a370a601258e1f9a95b3bc0975ed9f77","","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"47","abc.net.au","ABC Australia","Global news & media","Public broadcaster","Australia","Global","2026-10-21","https://www.abc.net.au/robots.txt","200","ok","2311","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Allowed","Blocked","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","net.au', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/abc.net.au.txt","751e4ba61018eed5ae3379dc5d244909782b50b21b03a9b307dd746670b7c573","751e4ba61018eed5ae3379dc5d244909782b50b21b03a9b307dd746670b7c573","Matches source index","https://abc.net.au/robots.txt","https://www.abc.net.au/robots.txt","","","faf6046e1eada5df3374fda9aa000887c6486553b84d73c7fa74e87ffd588009","No archived body","https://abc.net.au/llms.txt","https://www.abc.net.au/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/abc.net.au.txt","","751e4ba61018eed5ae3379dc5d244909782b50b21b03a9b307dd746670b7c573","","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Allowed","Blocked","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"48","smh.com.au","The Sydney Morning Herald","Global news & media","Newspaper","Australia","Global","2026-10-21","https://www.smh.com.au/robots.txt","200","ok","4048","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/smh.com.au.txt","15896d70e45f4ccd7ac931cb35e771740e2ffb74465fbf01bd01c6116acb1682","15896d70e45f4ccd7ac931cb35e771740e2ffb74465fbf01bd01c6116acb1682","Matches source index","https://smh.com.au/robots.txt","https://www.smh.com.au/robots.txt","","","779b8711328f5fd02b7dfd13ec8eea8ae614a7981680dbe697231db527db9c0b","No archived body","https://smh.com.au/llms.txt","https://www.smh.com.au/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/smh.com.au.txt","","15896d70e45f4ccd7ac931cb35e771740e2ffb74465fbf01bd01c6116acb1682","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"49","theage.com.au","The Age","Global news & media","Newspaper","Australia","Global","2026-10-21","https://www.theage.com.au/robots.txt","200","ok","4074","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/theage.com.au.txt","15a7f3f6313ea62b11e08190ffd641fb2547b286d723474dada013c037085369","15a7f3f6313ea62b11e08190ffd641fb2547b286d723474dada013c037085369","Matches source index","https://theage.com.au/robots.txt","https://www.theage.com.au/robots.txt","","","1860ab0f264653d4c163b1e7be6ebce3b947828072ddd16dcf4573ab2791ed1c","No archived body","https://theage.com.au/llms.txt","https://www.theage.com.au/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/theage.com.au.txt","","15a7f3f6313ea62b11e08190ffd641fb2547b286d723474dada013c037085369","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"50","stuff.co.nz","Stuff","Global news & media","Digital news","New Zealand","Global","2026-10-21","https://www.stuff.co.nz/robots.txt","200","ok","1620","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/stuff.co.nz.txt","b8e4e5866a826548199ad31b4ccb3286cb96547663b4f0b1ecccf00c07bd7694","b8e4e5866a826548199ad31b4ccb3286cb96547663b4f0b1ecccf00c07bd7694","Matches source index","https://stuff.co.nz/robots.txt","https://www.stuff.co.nz/robots.txt","llms/stuff.co.nz.txt","a62166806a3d89aef1b35ffe953bfdbc68ebe2e6c8a9926954503d461fac3f89","a62166806a3d89aef1b35ffe953bfdbc68ebe2e6c8a9926954503d461fac3f89","Matches source index","https://stuff.co.nz/llms.txt","https://www.stuff.co.nz/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/stuff.co.nz.txt","llms/stuff.co.nz.txt","b8e4e5866a826548199ad31b4ccb3286cb96547663b4f0b1ecccf00c07bd7694","a62166806a3d89aef1b35ffe953bfdbc68ebe2e6c8a9926954503d461fac3f89","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"51","nzherald.co.nz","The New Zealand Herald","Global news & media","Newspaper","New Zealand","Global","2026-10-21","https://www.nzherald.co.nz/robots.txt","200","ok","44665","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","co.nz', port=443): Read timed out. (read timeout=15)","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nzherald.co.nz.txt","367991479c11c2095bd019977959032197f9cf1208886e15cf80e2f77f828c74","367991479c11c2095bd019977959032197f9cf1208886e15cf80e2f77f828c74","Matches source index","https://nzherald.co.nz/robots.txt","https://www.nzherald.co.nz/robots.txt","llms/nzherald.co.nz.txt","fd058c5aad29e2af5d984d03d610d293b8898c22a190e9b63b0b6048c104d2bb","fd058c5aad29e2af5d984d03d610d293b8898c22a190e9b63b0b6048c104d2bb","Matches source index","https://nzherald.co.nz/llms.txt","https://www.nzherald.co.nz/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nzherald.co.nz.txt","llms/nzherald.co.nz.txt","367991479c11c2095bd019977959032197f9cf1208886e15cf80e2f77f828c74","fd058c5aad29e2af5d984d03d610d293b8898c22a190e9b63b0b6048c104d2bb","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"52","japantimes.co.jp","The Japan Times","Global news & media","Newspaper","Japan","Global","2026-10-21","https://www.japantimes.co.jp/robots.txt","200","ok","349","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","other","0","no_strict_links","362","True","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/japantimes.co.jp.txt","dd0aa72011ba46baf1d27868cfeb1e78049d1a8a97dffd9e99e57abc51869433","dd0aa72011ba46baf1d27868cfeb1e78049d1a8a97dffd9e99e57abc51869433","Matches source index","https://japantimes.co.jp/robots.txt","https://www.japantimes.co.jp/robots.txt","crawler-api-verification-pass/raw/llms/japantimes.co.jp--10222347-2533-0275-0000-ad87e021fd6d.txt","7be8380ae62f155be1300e9b2c387d6e405525a730c3774c0284cdc14d60ff43","","API-returned UTF-8 body; byte length matched provider size","https://japantimes.co.jp/llms.txt","https://www.japantimes.co.jp/","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/japantimes.co.jp.txt","","dd0aa72011ba46baf1d27868cfeb1e78049d1a8a97dffd9e99e57abc51869433","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","200","10222347-2533-0275-0000-ad87e021fd6d","2026-10-22 23:47:24 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"53","asahi.com","Asahi Shimbun","Global news & media","Newspaper","Japan","Global","2026-10-21","https://www.asahi.com/robots.txt","200","ok","2033","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/asahi.com.txt","327115416a3217db54213bbe6233b66996cf07640afb6d226749c4caaa759f8f","327115416a3217db54213bbe6233b66996cf07640afb6d226749c4caaa759f8f","Matches source index","https://asahi.com/robots.txt","https://www.asahi.com/robots.txt","","","708d161ba3c63cf4247d5bc1407ac6a63826c9474d3f32d867e68d4dfc961fa3","No archived body","https://asahi.com/llms.txt","https://www.asahi.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/asahi.com.txt","","327115416a3217db54213bbe6233b66996cf07640afb6d226749c4caaa759f8f","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"54","scmp.com","South China Morning Post","Global news & media","Newspaper","Hong Kong","Global","2026-10-22","https://www.scmp.com/robots.txt","200","ok","4518","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/scmp.com.txt","0bb4103be9dd02915567c7ef79abf72ae37568e571f554650dccc8ec7632a375","0bb4103be9dd02915567c7ef79abf72ae37568e571f554650dccc8ec7632a375","Matches source index","https://scmp.com/robots.txt","https://www.scmp.com/robots.txt","","","40cea97f8e57e041638cf70f2e4a26b3270d0cba905c2f88b74c5672ecf8d283","No archived body","https://scmp.com/llms.txt","https://www.scmp.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/scmp.com.txt","","0bb4103be9dd02915567c7ef79abf72ae37568e571f554650dccc8ec7632a375","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"55","straitstimes.com","The Straits Times","Global news & media","Newspaper","Singapore","Global","2026-10-21","https://www.straitstimes.com/robots.txt","200","ok","3237","valid_txt_200","True","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: straitstimes.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/straitstimes.com.txt","987d8feb3d47c749a6bdce9c715bb405cabd9c04daec97032ddbfeb0030c6cb7","987d8feb3d47c749a6bdce9c715bb405cabd9c04daec97032ddbfeb0030c6cb7","Matches source index","https://straitstimes.com/robots.txt","https://www.straitstimes.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://straitstimes.com/llms.txt","https://www.straitstimes.com/llms.txt","404","text/plain","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/straitstimes.com.txt","","987d8feb3d47c749a6bdce9c715bb405cabd9c04daec97032ddbfeb0030c6cb7","","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222341-2533-0275-0000-b5ba62619aca","2026-10-22 23:41:02 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-01.json","1"
"56","thehindu.com","The Hindu","Global news & media","Newspaper","India","Global","2026-10-21","https://www.thehindu.com/robots.txt","200","ok","4174","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","Baseline note:  llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/thehindu.com.txt","7e494e7fb330ced52571ffd93906b7c158d1e2966b363439333de9b4c96e05f9","7e494e7fb330ced52571ffd93906b7c158d1e2966b363439333de9b4c96e05f9","Matches source index","https://thehindu.com/robots.txt","https://www.thehindu.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://thehindu.com/llms.txt","https://www.thehindu.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/thehindu.com.txt","","7e494e7fb330ced52571ffd93906b7c158d1e2966b363439333de9b4c96e05f9","","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222353-2533-0275-0000-bb283b5c70e0","2026-10-22 23:53:54 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"57","hindustantimes.com","Hindustan Times","Global news & media","Newspaper","India","Global","2026-10-21","https://www.hindustantimes.com/robots.txt","200","ok","2449","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","41","has_strict_links","42","True","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/hindustantimes.com.txt","a2832bb6644409557e527fd01e6616ef5ab02cfc29b559ffb43030002dbc4d07","a2832bb6644409557e527fd01e6616ef5ab02cfc29b559ffb43030002dbc4d07","Matches source index","https://hindustantimes.com/robots.txt","https://www.hindustantimes.com/robots.txt","crawler-api-verification-pass/raw/llms/hindustantimes.com--10222341-2533-0275-0000-31f413818ed1.txt","d9c31a24f4ff6b9e6fe2eee2f8d01f090c018f85b185efe8a1075db0ce0f3412","","API-returned UTF-8 body; byte length matched provider size","https://hindustantimes.com/llms.txt","https://www.hindustantimes.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/hindustantimes.com.txt","","a2832bb6644409557e527fd01e6616ef5ab02cfc29b559ffb43030002dbc4d07","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222341-2533-0275-0000-31f413818ed1","2026-10-22 23:41:01 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-01.json","1"
"58","indianexpress.com","The Indian Express","Global news & media","Newspaper","India","Global","2026-10-21","https://indianexpress.com/robots.txt","200","ok","4779","valid_txt_200","True","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","True","True","False","unverified","","","","","","Baseline note:  llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Unverified","v3.2-verification","November 2026","raw/indianexpress.com.txt","fb2c63da04c5892792e6ad08ec236423b468727c38b102ecf7c6bb9f11d36a97","fb2c63da04c5892792e6ad08ec236423b468727c38b102ecf7c6bb9f11d36a97","Matches source index","https://indianexpress.com/robots.txt","https://indianexpress.com/robots.txt","","","","No archived body","https://indianexpress.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/indianexpress.com.txt","","fb2c63da04c5892792e6ad08ec236423b468727c38b102ecf7c6bb9f11d36a97","","Partial","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-65905f483534","2026-10-22 23:53:53 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"59","koreaherald.com","The Korea Herald","Global news & media","Newspaper","South Korea","Global","2026-10-21","https://www.koreaherald.com/robots.txt","200","ok","986","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/koreaherald.com.txt","893a03aecda211fc29c09156ba7da5a933520bc68eb9807f2b1d6d7a4d6d85ad","893a03aecda211fc29c09156ba7da5a933520bc68eb9807f2b1d6d7a4d6d85ad","Matches source index","https://koreaherald.com/robots.txt","https://www.koreaherald.com/robots.txt","","","3806b9be5f67c3aeb8257a3ac8d8e31cb96b8a9a58539ae233b7925dab77317d","No archived body","https://koreaherald.com/llms.txt","https://www.koreaherald.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/koreaherald.com.txt","","893a03aecda211fc29c09156ba7da5a933520bc68eb9807f2b1d6d7a4d6d85ad","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"60","bangkokpost.com","Bangkok Post","Global news & media","Newspaper","Thailand","Global","2026-10-21","https://www.bangkokpost.com/robots.txt","200","ok","539","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bangkokpost.com.txt","1d19aec0e15ed1e008545471ddee53b59c2d811f2231b04c83caae4815d30169","1d19aec0e15ed1e008545471ddee53b59c2d811f2231b04c83caae4815d30169","Matches source index","https://bangkokpost.com/robots.txt","https://www.bangkokpost.com/robots.txt","","","bcc880cd9504b5e8eb1742c596b36f3f4c9fc38940a207dc38df57477b314408","No archived body","https://bangkokpost.com/llms.txt","https://www.bangkokpost.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bangkokpost.com.txt","","1d19aec0e15ed1e008545471ddee53b59c2d811f2231b04c83caae4815d30169","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"61","microsoft.com","Microsoft","SaaS & technology platforms","Cloud platform","United States","Global","2026-10-21","https://www.microsoft.com/robots.txt","200","ok","9126","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/microsoft.com.txt","79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","Matches source index","https://microsoft.com/robots.txt","https://www.microsoft.com/robots.txt","","","6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","No archived body","https://microsoft.com/llms.txt","https://www.microsoft.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/microsoft.com.txt","","79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"62","google.com","Google","SaaS & technology platforms","Search & cloud platform","United States","Global","2026-10-21","https://www.google.com/robots.txt","200","ok","6624","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/google.com.txt","1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c","1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c","Matches source index","https://google.com/robots.txt","https://www.google.com/robots.txt","","","47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","No archived body","https://google.com/llms.txt","https://google.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/google.com.txt","","1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"63","apple.com","Apple","SaaS & technology platforms","Consumer technology platform","United States","Global","2026-10-21","https://www.apple.com/robots.txt","200","ok","1018","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/apple.com.txt","3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","Matches source index","https://apple.com/robots.txt","https://www.apple.com/robots.txt","","","2c33262e1cb35568064b86d703038e420bda53717cb88b78b7ff81e861ac5cde","No archived body","https://apple.com/llms.txt","https://www.apple.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/apple.com.txt","","3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"64","salesforce.com","Salesforce","SaaS & technology platforms","CRM platform","United States","Global","2026-10-21","https://www.salesforce.com/robots.txt","200","ok","14256","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","link_list_no_h1","3777","has_strict_links","3803","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/salesforce.com.txt","905d8800d5cc13bac380697922770b4c201bd32e570b858ff481cab029bbe20d","905d8800d5cc13bac380697922770b4c201bd32e570b858ff481cab029bbe20d","Matches source index","https://salesforce.com/robots.txt","https://www.salesforce.com/robots.txt","llms/salesforce.com.txt","c65c1ee7df6a23f391141700a7f00c086ec9530672a7e2a19934a9e33efb0e7f","c65c1ee7df6a23f391141700a7f00c086ec9530672a7e2a19934a9e33efb0e7f","Matches source index","https://salesforce.com/llms.txt","https://www.salesforce.com/llms.txt?bc=OTH","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/salesforce.com.txt","llms/salesforce.com.txt","905d8800d5cc13bac380697922770b4c201bd32e570b858ff481cab029bbe20d","c65c1ee7df6a23f391141700a7f00c086ec9530672a7e2a19934a9e33efb0e7f","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"65","adobe.com","Adobe","SaaS & technology platforms","Creative software","United States","Global","2026-10-22","https://www.adobe.com/robots.txt","200","ok","10199","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","46","has_strict_links","46","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/adobe.com.txt","41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","Matches source index","https://adobe.com/robots.txt","https://www.adobe.com/robots.txt","llms/adobe.com.txt","811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","Matches source index","https://adobe.com/llms.txt","https://www.adobe.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/adobe.com.txt","llms/adobe.com.txt","41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"66","oracle.com","Oracle","SaaS & technology platforms","Enterprise software & cloud","United States","Global","2026-10-21","https://www.oracle.com/robots.txt","200","ok","2649","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","oracle.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/oracle.com.txt","ce2fb919796ec17ecb2e8e928e4399beaf56c22d05e6cfd2018cb5aacf50ee71","ce2fb919796ec17ecb2e8e928e4399beaf56c22d05e6cfd2018cb5aacf50ee71","Matches source index","https://oracle.com/robots.txt","https://www.oracle.com/robots.txt","","","87955144891c54ebc577f3afbf0bf366da70c0014a90fd2c6899861c9b37b116","No archived body","https://oracle.com/llms.txt","https://www.oracle.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/oracle.com.txt","","ce2fb919796ec17ecb2e8e928e4399beaf56c22d05e6cfd2018cb5aacf50ee71","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"67","sap.com","SAP","SaaS & technology platforms","Enterprise software","Germany","Global","2026-10-21","https://www.sap.com/robots.txt","200","ok","4598","valid_txt_200","True","Partial","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","False","True","False","False","present_content_type_mismatch","h1_markdown","87","has_strict_links","87","True","sap.com', port=443): Read timed out. (read timeout=15) llms.txt: Markdown-like body served with text/html; excluded from strict-MIME presence count, not absent.","Usable response","Yes","None fully blocked","Present, MIME mismatch","v3.2-verification","November 2026","raw/sap.com.txt","79a44d145bd5ef1f5aa2b6a5f7ca9f266072a9d0e42962e4dc4632162ef292f6","79a44d145bd5ef1f5aa2b6a5f7ca9f266072a9d0e42962e4dc4632162ef292f6","Matches source index","https://sap.com/robots.txt","https://www.sap.com/robots.txt","llms/sap.com.txt","73290b066da58c5355e8a05b2942d4f964675e96f041dec3f7e37c937bc0b532","73290b066da58c5355e8a05b2942d4f964675e96f041dec3f7e37c937bc0b532","Matches source index","https://sap.com/llms.txt","https://www.sap.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present_content_type_mismatch","Present, MIME mismatch","raw/sap.com.txt","llms/sap.com.txt","79a44d145bd5ef1f5aa2b6a5f7ca9f266072a9d0e42962e4dc4632162ef292f6","73290b066da58c5355e8a05b2942d4f964675e96f041dec3f7e37c937bc0b532","Partial","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"68","ibm.com","IBM","SaaS & technology platforms","Enterprise technology","United States","Global","2026-10-21","https://www.ibm.com/robots.txt","200","ok","4616","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ibm.com.txt","8e29d2cf6f21b8d8eb24567c36e257fe1823582a6a9f3b2130410d8cc67f0f70","8e29d2cf6f21b8d8eb24567c36e257fe1823582a6a9f3b2130410d8cc67f0f70","Matches source index","https://ibm.com/robots.txt","https://www.ibm.com/robots.txt","","","73948440a68a8955787c14c57b3214113d86639b0ca4460cea9a1218f1a65150","No archived body","https://ibm.com/llms.txt","https://www.ibm.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ibm.com.txt","","8e29d2cf6f21b8d8eb24567c36e257fe1823582a6a9f3b2130410d8cc67f0f70","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"69","cisco.com","Cisco","SaaS & technology platforms","Networking technology","United States","Global","2026-10-22","https://www.cisco.com/robots.txt","200","ok","15619","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","link_list_no_h1","6","has_strict_links","6","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/cisco.com.txt","898698b0f990c8e1c2651d70d58cd1b92628985c07de0a1f6151d33ada24227c","898698b0f990c8e1c2651d70d58cd1b92628985c07de0a1f6151d33ada24227c","Matches source index","https://cisco.com/robots.txt","https://www.cisco.com/robots.txt","llms/cisco.com.txt","e47eb32547c6fb01757bfbdc9d99ec33516b91cb99284f1c56540b4651021545","e47eb32547c6fb01757bfbdc9d99ec33516b91cb99284f1c56540b4651021545","Matches source index","https://cisco.com/llms.txt","https://www.cisco.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/cisco.com.txt","llms/cisco.com.txt","898698b0f990c8e1c2651d70d58cd1b92628985c07de0a1f6151d33ada24227c","e47eb32547c6fb01757bfbdc9d99ec33516b91cb99284f1c56540b4651021545","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"70","intel.com","Intel","SaaS & technology platforms","Semiconductors","United States","Global","2026-10-22","https://www.intel.com/robots.txt","200","ok","4211","valid_txt_200","True","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","64","has_strict_links","64","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/intel.com.txt","930a465dbf2bd43fbcb048f3c092e13a036497efbaa86dee5bad4a8ec7219f8d","930a465dbf2bd43fbcb048f3c092e13a036497efbaa86dee5bad4a8ec7219f8d","Matches source index","https://intel.com/robots.txt","https://www.intel.com/robots.txt","llms/intel.com.txt","69845398118dd3d29170b557397b17dd8104d99225e4c10f48be60fc3b02fb96","69845398118dd3d29170b557397b17dd8104d99225e4c10f48be60fc3b02fb96","Matches source index","https://intel.com/llms.txt","https://www.intel.com/llms.txt","200","text/plain;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/intel.com.txt","llms/intel.com.txt","930a465dbf2bd43fbcb048f3c092e13a036497efbaa86dee5bad4a8ec7219f8d","69845398118dd3d29170b557397b17dd8104d99225e4c10f48be60fc3b02fb96","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"71","nvidia.com","NVIDIA","SaaS & technology platforms","Semiconductors & computing","United States","Global","2026-10-21","https://www.nvidia.com/robots.txt","200","ok","3725","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","19","has_strict_links","20","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/nvidia.com.txt","5069f0e6713ea599a59d48d5fb160ac859af14dcce9ec2a87fb9f554fe0f7125","5069f0e6713ea599a59d48d5fb160ac859af14dcce9ec2a87fb9f554fe0f7125","Matches source index","https://nvidia.com/robots.txt","https://www.nvidia.com/robots.txt","llms/nvidia.com.txt","069e726170096ccb0357322dc6d66d6cc9680914903367dd0f1b60a2c0b163d2","069e726170096ccb0357322dc6d66d6cc9680914903367dd0f1b60a2c0b163d2","Matches source index","https://nvidia.com/llms.txt","https://www.nvidia.com/llms.txt","200","text/markdown; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/nvidia.com.txt","llms/nvidia.com.txt","5069f0e6713ea599a59d48d5fb160ac859af14dcce9ec2a87fb9f554fe0f7125","069e726170096ccb0357322dc6d66d6cc9680914903367dd0f1b60a2c0b163d2","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"72","dell.com","Dell Technologies","SaaS & technology platforms","Computer technology","United States","Global","2026-10-21","https://www.dell.com/robots.txt","200","ok","28897","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","284","has_strict_links","285","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/dell.com.txt","66cd4b30830097a49824ea5ca499411193336ccae283787b385bc3b43f620bf4","66cd4b30830097a49824ea5ca499411193336ccae283787b385bc3b43f620bf4","Matches source index","https://dell.com/robots.txt","https://www.dell.com/robots.txt","llms/dell.com.txt","7b4d53bd18f07ea2f0655959ad085346fd0766f36791def7b941eebc1bf7b16f","7b4d53bd18f07ea2f0655959ad085346fd0766f36791def7b941eebc1bf7b16f","Matches source index","https://dell.com/llms.txt","https://www.dell.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/dell.com.txt","llms/dell.com.txt","66cd4b30830097a49824ea5ca499411193336ccae283787b385bc3b43f620bf4","7b4d53bd18f07ea2f0655959ad085346fd0766f36791def7b941eebc1bf7b16f","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"73","hp.com","HP","SaaS & technology platforms","Computer technology","United States","Global","2026-10-21","https://www.hp.com/robots.txt","200","ok","17151","valid_txt_200","True","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hp.com.txt","05a4d45bf193432243653fafacaf4fbd13926550a76711d0c09d8a996f83727c","05a4d45bf193432243653fafacaf4fbd13926550a76711d0c09d8a996f83727c","Matches source index","https://hp.com/robots.txt","https://www.hp.com/robots.txt","","","63c0de7c6c7aea5753ef7db15d995b9046322af26b640d6458456e0d2a932d80","No archived body","https://hp.com/llms.txt","https://www.hp.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hp.com.txt","","05a4d45bf193432243653fafacaf4fbd13926550a76711d0c09d8a996f83727c","","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"74","lenovo.com","Lenovo","SaaS & technology platforms","Computer technology","China","Global","2026-10-22","https://www.lenovo.com/robots.txt","200","ok","3633","valid_txt_200","True","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Partial","Allowed","Partial","False","False","False","False","present","h1_markdown","34","has_strict_links","34","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/lenovo.com.txt","65d1fb446b8b9ba30eba53bb2b46a718c486046e61774eadd17b17cb51304482","65d1fb446b8b9ba30eba53bb2b46a718c486046e61774eadd17b17cb51304482","Matches source index","https://lenovo.com/robots.txt","https://www.lenovo.com/robots.txt","llms/lenovo.com.txt","767d784b9119ca1bf04cee136e4523f8f3f9d408ef47e15847008dab6dce0be9","767d784b9119ca1bf04cee136e4523f8f3f9d408ef47e15847008dab6dce0be9","Matches source index","https://lenovo.com/llms.txt","https://www.lenovo.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/lenovo.com.txt","llms/lenovo.com.txt","65d1fb446b8b9ba30eba53bb2b46a718c486046e61774eadd17b17cb51304482","767d784b9119ca1bf04cee136e4523f8f3f9d408ef47e15847008dab6dce0be9","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Partial","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"75","samsung.com","Samsung","SaaS & technology platforms","Consumer technology","South Korea","Global","2026-10-22","https://www.samsung.com/robots.txt","200","ok","10085","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","31","has_strict_links","31","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/samsung.com.txt","e07ce06f3220d0496ee9bec373ddb2e2fdc272391a4b0153c453bd39b08e39f8","e07ce06f3220d0496ee9bec373ddb2e2fdc272391a4b0153c453bd39b08e39f8","Matches source index","https://samsung.com/robots.txt","https://www.samsung.com/robots.txt","llms/samsung.com.txt","d248d7acb33e01eff77c0c2fe1b2f25196ce8d9265ce133583435a80b686589f","d248d7acb33e01eff77c0c2fe1b2f25196ce8d9265ce133583435a80b686589f","Matches source index","https://samsung.com/llms.txt","https://www.samsung.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/samsung.com.txt","llms/samsung.com.txt","e07ce06f3220d0496ee9bec373ddb2e2fdc272391a4b0153c453bd39b08e39f8","d248d7acb33e01eff77c0c2fe1b2f25196ce8d9265ce133583435a80b686589f","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"76","sony.com","Sony","SaaS & technology platforms","Consumer technology","Japan","Global","2026-10-22","https://sony.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://sony.com/robots.txt","https://sony.com/robots.txt","","","76f587726867a138642bce19851f1e53248c4bea63ccc083a6252c0f1a7d03d6","No archived body","https://sony.com/llms.txt","https://www.sony.com:443/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222347-2533-0275-0000-0741430db765","2026-10-22 23:47:22 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-90839105e5ed","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"77","atlassian.com","Atlassian","SaaS & technology platforms","Collaboration software","Australia","Global","2026-10-21","https://www.atlassian.com/robots.txt","200","ok","1292","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","105","has_strict_links","105","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/atlassian.com.txt","c0a6d137d17b75f9f916b616a77e87fde588b0adb4338127b03b55ea77ba3841","c0a6d137d17b75f9f916b616a77e87fde588b0adb4338127b03b55ea77ba3841","Matches source index","https://atlassian.com/robots.txt","https://www.atlassian.com/robots.txt","llms/atlassian.com.txt","cd8609813ba417ea3040411a22551a91b7663f552724332c97758649bba82ca2","cd8609813ba417ea3040411a22551a91b7663f552724332c97758649bba82ca2","Matches source index","https://atlassian.com/llms.txt","https://www.atlassian.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/atlassian.com.txt","llms/atlassian.com.txt","c0a6d137d17b75f9f916b616a77e87fde588b0adb4338127b03b55ea77ba3841","cd8609813ba417ea3040411a22551a91b7663f552724332c97758649bba82ca2","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"78","slack.com","Slack","SaaS & technology platforms","Collaboration software","United States","Global","2026-10-21","https://slack.com/robots.txt","200","ok","484","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","165","has_strict_links","165","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/slack.com.txt","31d812318ba1ef6234667f41c8d97cf1bb62b4173915eaa5a2c24d4e60a07fe6","31d812318ba1ef6234667f41c8d97cf1bb62b4173915eaa5a2c24d4e60a07fe6","Matches source index","https://slack.com/robots.txt","https://slack.com/robots.txt","llms/slack.com.txt","d08ba41e1e7bbccb335dd566f975bb9e93ad9f65bfa48a5c0fb109bc0095e267","d08ba41e1e7bbccb335dd566f975bb9e93ad9f65bfa48a5c0fb109bc0095e267","Matches source index","https://slack.com/llms.txt","https://slack.com/llms.txt","200","text/plain;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/slack.com.txt","llms/slack.com.txt","31d812318ba1ef6234667f41c8d97cf1bb62b4173915eaa5a2c24d4e60a07fe6","d08ba41e1e7bbccb335dd566f975bb9e93ad9f65bfa48a5c0fb109bc0095e267","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"79","zoom.us","Zoom","SaaS & technology platforms","Video communications","United States","Global","2026-10-21","https://zoom.us/robots.txt","200","ok","2005","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/zoom.us.txt","b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","Matches source index","https://zoom.us/robots.txt","https://zoom.us/robots.txt","","","13a80f9db7bc639a083e74897b92d67ff528e2a4271e6042450091d1435db1a7","No archived body","https://zoom.us/llms.txt","https://zoom.us/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/zoom.us.txt","","b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"80","dropbox.com","Dropbox","SaaS & technology platforms","Cloud storage","United States","Global","2026-10-21","https://www.dropbox.com/robots.txt","200","ok","801","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","829","has_strict_links","829","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/dropbox.com.txt","cc244a992405a4c1368861e7a1968a68ae1d48567b5802fe4d1f1e00d73974b3","cc244a992405a4c1368861e7a1968a68ae1d48567b5802fe4d1f1e00d73974b3","Matches source index","https://dropbox.com/robots.txt","https://www.dropbox.com/robots.txt","llms/dropbox.com.txt","ad9ed13f7ca90a0a91d36799eac5aa2bf9bdc985e66ddf8b8583f83a30556171","ad9ed13f7ca90a0a91d36799eac5aa2bf9bdc985e66ddf8b8583f83a30556171","Matches source index","https://dropbox.com/llms.txt","https://www.dropbox.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/dropbox.com.txt","llms/dropbox.com.txt","cc244a992405a4c1368861e7a1968a68ae1d48567b5802fe4d1f1e00d73974b3","ad9ed13f7ca90a0a91d36799eac5aa2bf9bdc985e66ddf8b8583f83a30556171","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"81","box.com","Box","SaaS & technology platforms","Cloud content platform","United States","Global","2026-10-21","https://www.box.com/robots.txt","200","ok","1484","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","2720","has_strict_links","2720","True","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/box.com.txt","cb77a92c0a2fe2e45eb81e381bda7c843c0688e0066cd2274fb06a0644deffa9","cb77a92c0a2fe2e45eb81e381bda7c843c0688e0066cd2274fb06a0644deffa9","Matches source index","https://box.com/robots.txt","https://www.box.com/robots.txt","crawler-api-verification-pass/raw/llms/box.com--10222341-2533-0275-0000-f943d4f4f600.txt","e453df2ffe2ac0d9426d22bc933f913514d5707b1d70e09268f9ea6ec1ed11fb","","API-returned UTF-8 body; byte length matched provider size","https://box.com/llms.txt","https://www.box.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/box.com.txt","","cb77a92c0a2fe2e45eb81e381bda7c843c0688e0066cd2274fb06a0644deffa9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222341-2533-0275-0000-f943d4f4f600","2026-10-22 23:41:03 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-01.json","1"
"82","servicenow.com","ServiceNow","SaaS & technology platforms","Workflow platform","United States","Global","2026-10-22","https://servicenow.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 servicenow.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://servicenow.com/robots.txt","https://servicenow.com/robots.txt","","","9ba7be897cc07a7a44e5e261b65db7362afc4bd8d7c5d9650215ab1f0c58eb52","No archived body","https://servicenow.com/llms.txt","https://www.servicenow.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222347-2533-0275-0000-1351be2793be","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-pilot.json","2","Frozen v3.1.2 baseline","Undetermined","","10222353-2533-0275-0000-1650860a011a","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"83","workday.com","Workday","SaaS & technology platforms","HR & finance software","United States","Global","2026-10-21","https://www.workday.com/robots.txt","200","ok","6757","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/workday.com.txt","d3d177d18a12e1ee7f68af1f49284b655fd4e69bb9377823bf96417f62e03328","d3d177d18a12e1ee7f68af1f49284b655fd4e69bb9377823bf96417f62e03328","Matches source index","https://workday.com/robots.txt","https://www.workday.com/robots.txt","","","89b3d01b14a19a4dd67fd526d1d9a48e98e90fed61b5718ebc88cc50bee52991","No archived body","https://workday.com/llms.txt","https://www.workday.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/workday.com.txt","","d3d177d18a12e1ee7f68af1f49284b655fd4e69bb9377823bf96417f62e03328","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"84","intuit.com","Intuit","SaaS & technology platforms","Financial software","United States","Global","2026-10-21","https://www.intuit.com/robots.txt","200","ok","3086","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","47","has_strict_links","49","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/intuit.com.txt","99f9354d3dc7685b0a7c25eb9370fb367e240013a28fd6600bf705984c5c1444","99f9354d3dc7685b0a7c25eb9370fb367e240013a28fd6600bf705984c5c1444","Matches source index","https://intuit.com/robots.txt","https://www.intuit.com/robots.txt","llms/intuit.com.txt","23db7511adac8d695410917aa5bdd7e2cad0fd29ef382968db19194a0a37d57c","23db7511adac8d695410917aa5bdd7e2cad0fd29ef382968db19194a0a37d57c","Matches source index","https://intuit.com/llms.txt","https://www.intuit.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/intuit.com.txt","llms/intuit.com.txt","99f9354d3dc7685b0a7c25eb9370fb367e240013a28fd6600bf705984c5c1444","23db7511adac8d695410917aa5bdd7e2cad0fd29ef382968db19194a0a37d57c","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"85","autodesk.com","Autodesk","SaaS & technology platforms","Design software","United States","Global","2026-10-22","https://autodesk.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://autodesk.com/robots.txt","https://autodesk.com/robots.txt","","","06029048daaf2fddb08f3c68b5236fad57ee9a4c39def104719af30f971ec26a","No archived body","https://autodesk.com/llms.txt","https://www.autodesk.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222347-2533-0275-0000-a0906cae2baf","2026-10-22 23:47:22 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-9c70102c4a71","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"86","vmware.com","VMware","SaaS & technology platforms","Virtualisation software","United States","Global","2026-10-21","https://www.vmware.com/robots.txt","200","ok","477","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","vmware.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/vmware.com.txt","daca8b7be2926bfdaaa7db56d9fa74aed2bf2129cf3f92b49789ca77490bb1cf","daca8b7be2926bfdaaa7db56d9fa74aed2bf2129cf3f92b49789ca77490bb1cf","Matches source index","https://vmware.com/robots.txt","https://www.vmware.com/robots.txt","","","ef6b031702b275aa9d61942a555f532d6590a91dc8ef07a38db91931b666231d","No archived body","https://www.vmware.com/llms.txt","https://www.vmware.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/vmware.com.txt","","daca8b7be2926bfdaaa7db56d9fa74aed2bf2129cf3f92b49789ca77490bb1cf","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"87","redhat.com","Red Hat","SaaS & technology platforms","Open-source software","United States","Global","2026-10-21","https://www.redhat.com/robots.txt","200","ok","3368","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","28","has_strict_links","28","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/redhat.com.txt","b0c016632108d71da6be63b5dcfa05c5e0944adf9c5c91959736b9b2f8f58990","b0c016632108d71da6be63b5dcfa05c5e0944adf9c5c91959736b9b2f8f58990","Matches source index","https://redhat.com/robots.txt","https://www.redhat.com/robots.txt","llms/redhat.com.txt","98be965f653943d2dc844f6465b728228a829584aa64da0a642a72dbdc01f083","98be965f653943d2dc844f6465b728228a829584aa64da0a642a72dbdc01f083","Matches source index","https://redhat.com/llms.txt","https://www.redhat.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/redhat.com.txt","llms/redhat.com.txt","b0c016632108d71da6be63b5dcfa05c5e0944adf9c5c91959736b9b2f8f58990","98be965f653943d2dc844f6465b728228a829584aa64da0a642a72dbdc01f083","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"88","gitlab.com","GitLab","SaaS & technology platforms","DevOps platform","United States","Global","2026-10-21","https://gitlab.com/robots.txt","200","ok","3230","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/gitlab.com.txt","9e1e837e03b726eedb04c30b565e69068ccbb96c4d12c7c090d9affcf54f5eb0","9e1e837e03b726eedb04c30b565e69068ccbb96c4d12c7c090d9affcf54f5eb0","Matches source index","https://gitlab.com/robots.txt","https://gitlab.com/robots.txt","","","b011f6185b09114f094e5ff4d135493d931ede3c71a421b65bbe10bf38f93206","No archived body","https://gitlab.com/llms.txt","https://gitlab.com/users/sign_in","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/gitlab.com.txt","","9e1e837e03b726eedb04c30b565e69068ccbb96c4d12c7c090d9affcf54f5eb0","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222347-2533-0275-0000-d24a4bbfe90f","2026-10-22 23:47:22 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"89","docker.com","Docker","SaaS & technology platforms","Developer platform","United States","Global","2026-10-21","https://www.docker.com/robots.txt","200","ok","1055","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/docker.com.txt","d0897a8a1636c06f2da7c5a3c6ed4a970b2c5565934bd02a0b48af5c3b99a9e0","d0897a8a1636c06f2da7c5a3c6ed4a970b2c5565934bd02a0b48af5c3b99a9e0","Matches source index","https://docker.com/robots.txt","https://www.docker.com/robots.txt","","","10d5c198c559fd64a94e15591750477737c728e3ff3c725f0115d7e0b4646ce1","No archived body","https://docker.com/llms.txt","https://www.docker.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/docker.com.txt","","d0897a8a1636c06f2da7c5a3c6ed4a970b2c5565934bd02a0b48af5c3b99a9e0","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"90","cloudflare.com","Cloudflare","SaaS & technology platforms","Web infrastructure","United States","Global","2026-10-21","https://www.cloudflare.com/robots.txt","200","ok","1099","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","90","has_strict_links","90","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/cloudflare.com.txt","c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","Matches source index","https://cloudflare.com/robots.txt","https://www.cloudflare.com/robots.txt","llms/cloudflare.com.txt","81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","Matches source index","https://cloudflare.com/llms.txt","https://www.cloudflare.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/cloudflare.com.txt","llms/cloudflare.com.txt","c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"91","akamai.com","Akamai","SaaS & technology platforms","Web infrastructure","United States","Global","2026-10-22","https://akamai.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://akamai.com/robots.txt","https://akamai.com/robots.txt","","","4372805954ec888bb1475ff54aa0fa6a38f9fd9135e6889ae965a87fffd834c6","No archived body","https://akamai.com/llms.txt","https://www.akamai.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-d4c8f7c5db97","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-eb07a9b7bf6c","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"92","netflix.com","Netflix","SaaS & technology platforms","Streaming platform","United States","Global","2026-10-21","https://www.netflix.com/robots.txt","200","ok","3790","valid_txt_200","True","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","False","True","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/netflix.com.txt","1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","Matches source index","https://netflix.com/robots.txt","https://www.netflix.com/robots.txt","llms/netflix.com.txt","12fa1702f2767fd2d947b1caf59f6cac730e7cb764ad191bbca0229f10e4602d","12fa1702f2767fd2d947b1caf59f6cac730e7cb764ad191bbca0229f10e4602d","Matches source index","https://netflix.com/llms.txt","https://www.netflix.com/NotFound?prev=https%3A%2F%2Fwww.netflix.com%2Fllms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/netflix.com.txt","llms/netflix.com.txt","1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","12fa1702f2767fd2d947b1caf59f6cac730e7cb764ad191bbca0229f10e4602d","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"93","spotify.com","Spotify","SaaS & technology platforms","Audio streaming platform","Sweden","Global","2026-10-21","https://www.spotify.com/robots.txt","200","ok","635","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Max retries exceeded with url: /","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/spotify.com.txt","b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","Matches source index","https://spotify.com/robots.txt","https://www.spotify.com/robots.txt","llms/spotify.com.txt","64e54130cc9cb16954b795b1d89207de7269584f2d439ab16dccada0390d3174","64e54130cc9cb16954b795b1d89207de7269584f2d439ab16dccada0390d3174","Matches source index","https://spotify.com/llms.txt","https://www.spotify.com/pk-en/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/spotify.com.txt","llms/spotify.com.txt","b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","64e54130cc9cb16954b795b1d89207de7269584f2d439ab16dccada0390d3174","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"94","uber.com","Uber","SaaS & technology platforms","Mobility platform","United States","Global","2026-10-21","https://www.uber.com/robots.txt","200","ok","904","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/uber.com.txt","40a0335cf0146087d2473b20e8d2c437406e8eaf0bcd4ed93c24c874a1d560db","40a0335cf0146087d2473b20e8d2c437406e8eaf0bcd4ed93c24c874a1d560db","Matches source index","https://uber.com/robots.txt","https://www.uber.com/robots.txt","","","0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","No archived body","https://uber.com/llms.txt","https://www.uber.com/llms.txt","404","text/plain; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/uber.com.txt","","40a0335cf0146087d2473b20e8d2c437406e8eaf0bcd4ed93c24c874a1d560db","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"95","snowflake.com","Snowflake","SaaS & technology platforms","Data cloud platform","United States","Global","2026-10-21","https://www.snowflake.com/robots.txt","200","ok","2751","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/snowflake.com.txt","8f7044965766cc80e5fe2cff08f94e704fc7f5902378c4fe59c1afd2ef55a384","8f7044965766cc80e5fe2cff08f94e704fc7f5902378c4fe59c1afd2ef55a384","Matches source index","https://snowflake.com/robots.txt","https://www.snowflake.com/robots.txt","","","a8b8034b90af886723af44de65e8e5ce1b7c079129a16e8c3915019a0535fe79","No archived body","https://snowflake.com/llms.txt","https://www.snowflake.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/snowflake.com.txt","","8f7044965766cc80e5fe2cff08f94e704fc7f5902378c4fe59c1afd2ef55a384","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"96","datadoghq.com","Datadog","SaaS & technology platforms","Monitoring platform","United States","Global","2026-10-21","https://www.datadoghq.com/robots.txt","200","ok","525","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","2410","has_strict_links","2410","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/datadoghq.com.txt","31eeea994b43f6d7f90690150759601a19c7a76f19827a51b18c89c6cbb1076b","31eeea994b43f6d7f90690150759601a19c7a76f19827a51b18c89c6cbb1076b","Matches source index","https://datadoghq.com/robots.txt","https://www.datadoghq.com/robots.txt","llms/datadoghq.com.txt","e915a01dc386ec90775ad658c1345a3577110c1342cdc74ce04640d211651795","e915a01dc386ec90775ad658c1345a3577110c1342cdc74ce04640d211651795","Matches source index","https://datadoghq.com/llms.txt","https://www.datadoghq.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/datadoghq.com.txt","llms/datadoghq.com.txt","31eeea994b43f6d7f90690150759601a19c7a76f19827a51b18c89c6cbb1076b","e915a01dc386ec90775ad658c1345a3577110c1342cdc74ce04640d211651795","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"97","mongodb.com","MongoDB","SaaS & technology platforms","Database platform","United States","Global","2026-10-21","https://www.mongodb.com/robots.txt","200","ok","2859","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","64","has_strict_links","64","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/mongodb.com.txt","79b3e7543ec8c0e56527f4cd232f1ee5f8c6872ea0c10c74820f0f64822a71f8","79b3e7543ec8c0e56527f4cd232f1ee5f8c6872ea0c10c74820f0f64822a71f8","Matches source index","https://mongodb.com/robots.txt","https://www.mongodb.com/robots.txt","llms/mongodb.com.txt","673298d5ce2c1bd2f33c70b4ef4655bb9018de3a9c300336d23e69a281823369","673298d5ce2c1bd2f33c70b4ef4655bb9018de3a9c300336d23e69a281823369","Matches source index","https://mongodb.com/llms.txt","https://www.mongodb.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/mongodb.com.txt","llms/mongodb.com.txt","79b3e7543ec8c0e56527f4cd232f1ee5f8c6872ea0c10c74820f0f64822a71f8","673298d5ce2c1bd2f33c70b4ef4655bb9018de3a9c300336d23e69a281823369","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"98","twilio.com","Twilio","SaaS & technology platforms","Communications platform","United States","Global","2026-10-21","https://www.twilio.com:443/robots.txt","200","ok","976","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","link_list_no_h1","7501","has_strict_links","7512","True","robots.txt redirects to host www.twilio.com:443","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/twilio.com.txt","8a61d4ade19db83d41254c1a0b46f012c604985193e9aceb127a90be39407525","8a61d4ade19db83d41254c1a0b46f012c604985193e9aceb127a90be39407525","Matches source index","https://twilio.com/robots.txt","https://www.twilio.com:443/robots.txt","llms/twilio.com.txt","2b151651e5b0f65c4e1587d686017d6a6cf78acf8b1aa4c42cc07c2e1a823eea","2b151651e5b0f65c4e1587d686017d6a6cf78acf8b1aa4c42cc07c2e1a823eea","Matches source index","https://twilio.com/llms.txt","https://www.twilio.com:443/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/twilio.com.txt","llms/twilio.com.txt","8a61d4ade19db83d41254c1a0b46f012c604985193e9aceb127a90be39407525","2b151651e5b0f65c4e1587d686017d6a6cf78acf8b1aa4c42cc07c2e1a823eea","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"99","zendesk.com","Zendesk","SaaS & technology platforms","Customer service software","United States","Global","2026-10-21","https://www.zendesk.com/robots.txt","200","ok","1028","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","3136","has_strict_links","3136","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/zendesk.com.txt","febf032bdcc18f83dbd17c75b2903f569c43c3f21f364616b2cda97714c36166","febf032bdcc18f83dbd17c75b2903f569c43c3f21f364616b2cda97714c36166","Matches source index","https://zendesk.com/robots.txt","https://www.zendesk.com/robots.txt","llms/zendesk.com.txt","047693a72525ecfe59837a95a2e6c961fd847faeb3f21fa80473f8fd9696a0f1","047693a72525ecfe59837a95a2e6c961fd847faeb3f21fa80473f8fd9696a0f1","Matches source index","https://zendesk.com/llms.txt","https://www.zendesk.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/zendesk.com.txt","llms/zendesk.com.txt","febf032bdcc18f83dbd17c75b2903f569c43c3f21f364616b2cda97714c36166","047693a72525ecfe59837a95a2e6c961fd847faeb3f21fa80473f8fd9696a0f1","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"100","asana.com","Asana","SaaS & technology platforms","Work management software","United States","Global","2026-10-21","https://asana.com/robots.txt","200","ok","547","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","44","has_strict_links","44","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/asana.com.txt","1b69de0d3887cd5568f7df92f245087d9e815547f68241146797e0a2562b1649","1b69de0d3887cd5568f7df92f245087d9e815547f68241146797e0a2562b1649","Matches source index","https://asana.com/robots.txt","https://asana.com/robots.txt","llms/asana.com.txt","37b3d59783f8b1fcda7353631529ecb15068e3677d1c615f6044b105ce5035b6","37b3d59783f8b1fcda7353631529ecb15068e3677d1c615f6044b105ce5035b6","Matches source index","https://asana.com/llms.txt","https://asana.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/asana.com.txt","llms/asana.com.txt","1b69de0d3887cd5568f7df92f245087d9e815547f68241146797e0a2562b1649","37b3d59783f8b1fcda7353631529ecb15068e3677d1c615f6044b105ce5035b6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"101","monday.com","monday.com","SaaS & technology platforms","Work management software","Israel","Global","2026-10-21","https://monday.com/robots.txt","200","ok","3091","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","53","has_strict_links","54","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/monday.com.txt","2dfc4fa504381ce292062740f5cc0c42c6711b0113fdca6cbe1e648e8cb3cbb1","2dfc4fa504381ce292062740f5cc0c42c6711b0113fdca6cbe1e648e8cb3cbb1","Matches source index","https://monday.com/robots.txt","https://monday.com/robots.txt","llms/monday.com.txt","a1de7f7d23f8c1eff99f9e6696bb8547876231d700b3ecb4ae7a4aee3f523366","a1de7f7d23f8c1eff99f9e6696bb8547876231d700b3ecb4ae7a4aee3f523366","Matches source index","https://monday.com/llms.txt","https://monday.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/monday.com.txt","llms/monday.com.txt","2dfc4fa504381ce292062740f5cc0c42c6711b0113fdca6cbe1e648e8cb3cbb1","a1de7f7d23f8c1eff99f9e6696bb8547876231d700b3ecb4ae7a4aee3f523366","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"102","notion.so","Notion","SaaS & technology platforms","Productivity software","United States","Global","2026-10-21","https://www.notion.so/robots.txt","200","ok","1136","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","link_list_no_h1","48","has_strict_links","48","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/notion.so.txt","f4a0f8257adb7d21c0ca560855d756d848f87eab9091fadb45005910f893f113","f4a0f8257adb7d21c0ca560855d756d848f87eab9091fadb45005910f893f113","Matches source index","https://notion.so/robots.txt","https://www.notion.so/robots.txt","llms/notion.so.txt","831abe03df610af9849fce8c5e52836b6cb59962255a98bf95a6eabe35edc754","831abe03df610af9849fce8c5e52836b6cb59962255a98bf95a6eabe35edc754","Matches source index","https://notion.so/llms.txt","https://www.notion.so/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/notion.so.txt","llms/notion.so.txt","f4a0f8257adb7d21c0ca560855d756d848f87eab9091fadb45005910f893f113","831abe03df610af9849fce8c5e52836b6cb59962255a98bf95a6eabe35edc754","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"103","figma.com","Figma","SaaS & technology platforms","Design software","United States","Global","2026-10-21","https://www.figma.com/robots.txt","200","ok","383816","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/figma.com.txt","e2a14ffff5825bddb1186a80770c27896f30014c2f82a525d90a3203f2848de6","e2a14ffff5825bddb1186a80770c27896f30014c2f82a525d90a3203f2848de6","Matches source index","https://figma.com/robots.txt","https://www.figma.com/robots.txt","","","101f6982766aa69f39d9b81b93278dff9b0d77d5ad76270b0b74d38986e9838d","No archived body","https://figma.com/llms.txt","https://www.figma.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/figma.com.txt","","e2a14ffff5825bddb1186a80770c27896f30014c2f82a525d90a3203f2848de6","","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"104","canva.com","Canva","SaaS & technology platforms","Design platform","Australia","Global","2026-10-21","https://www.canva.com/robots.txt","200","ok","6527","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/canva.com.txt","394d1f7e06d1445dd97b77bf3c53424e1288301257956f4b503ecb9c7833efdb","394d1f7e06d1445dd97b77bf3c53424e1288301257956f4b503ecb9c7833efdb","Matches source index","https://canva.com/robots.txt","https://www.canva.com/robots.txt","","","4e864d7d64d1aa3c3136dbcf8fc9b585ef6beb0b633f4c10ad705296d81cc1d2","No archived body","https://canva.com/llms.txt","https://www.canva.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/canva.com.txt","","394d1f7e06d1445dd97b77bf3c53424e1288301257956f4b503ecb9c7833efdb","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"105","miro.com","Miro","SaaS & technology platforms","Collaboration software","United States","Global","2026-10-21","https://miro.com/robots.txt","200","ok","1703","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/miro.com.txt","0da5ef88c9ee190100bea060fd4243e9dfe1ada804103e7ea8d1d99a35cc76be","0da5ef88c9ee190100bea060fd4243e9dfe1ada804103e7ea8d1d99a35cc76be","Matches source index","https://miro.com/robots.txt","https://miro.com/robots.txt","","","5e96b2b119fe12f0dc1e8008b10e244fba504ebcc1e47cbc828d3c60a3f71b0a","No archived body","https://miro.com/llms.txt","https://miro.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/miro.com.txt","","0da5ef88c9ee190100bea060fd4243e9dfe1ada804103e7ea8d1d99a35cc76be","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"106","airtable.com","Airtable","SaaS & technology platforms","Database & workflow software","United States","Global","2026-10-21","https://airtable.com/robots.txt","200","ok","506","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/airtable.com.txt","5fba1f6452a8bc57e0c091c362ba8f886846a5d7eba2de5ee85c72402171bb4c","5fba1f6452a8bc57e0c091c362ba8f886846a5d7eba2de5ee85c72402171bb4c","Matches source index","https://airtable.com/robots.txt","https://airtable.com/robots.txt","","","6557b88a81fb9a979f8b5b59b88e1a914f8fa4af0d4423cb5f7d771ab012d1df","No archived body","https://airtable.com/llms.txt","https://airtable.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/airtable.com.txt","","5fba1f6452a8bc57e0c091c362ba8f886846a5d7eba2de5ee85c72402171bb4c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"107","webflow.com","Webflow","SaaS & technology platforms","Website platform","United States","Global","2026-10-22","https://webflow.com/robots.txt","200","ok","279","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","76","has_strict_links","78","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/webflow.com.txt","b04ed9b180086889a2046dc5e952815a6b642497a787f5b54c698cd2549e5d98","b04ed9b180086889a2046dc5e952815a6b642497a787f5b54c698cd2549e5d98","Matches source index","https://webflow.com/robots.txt","https://webflow.com/robots.txt","llms/webflow.com.txt","7cec56881f0b2b2be8dc4878f3007fdfb881246ae38ac2fb501b2619c01d319d","7cec56881f0b2b2be8dc4878f3007fdfb881246ae38ac2fb501b2619c01d319d","Matches source index","https://webflow.com/llms.txt","https://webflow.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/webflow.com.txt","llms/webflow.com.txt","b04ed9b180086889a2046dc5e952815a6b642497a787f5b54c698cd2549e5d98","7cec56881f0b2b2be8dc4878f3007fdfb881246ae38ac2fb501b2619c01d319d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"108","wix.com","Wix","SaaS & technology platforms","Website platform","Israel","Global","2026-10-21","https://www.wix.com/robots.txt","200","ok","2797","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","232","has_strict_links","232","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/wix.com.txt","fef1f6ed42a555f0eb1efa45b48d3d00c98d8eb0452bb6d2a1487d6ba8771d6a","fef1f6ed42a555f0eb1efa45b48d3d00c98d8eb0452bb6d2a1487d6ba8771d6a","Matches source index","https://wix.com/robots.txt","https://www.wix.com/robots.txt","llms/wix.com.txt","ccfa8dc4eb331c658b95d69579dcb70aff2785b51af56ec94fd63e4455c3d529","ccfa8dc4eb331c658b95d69579dcb70aff2785b51af56ec94fd63e4455c3d529","Matches source index","https://wix.com/llms.txt","https://www.wix.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/wix.com.txt","llms/wix.com.txt","fef1f6ed42a555f0eb1efa45b48d3d00c98d8eb0452bb6d2a1487d6ba8771d6a","ccfa8dc4eb331c658b95d69579dcb70aff2785b51af56ec94fd63e4455c3d529","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"109","squarespace.com","Squarespace","SaaS & technology platforms","Website platform","United States","Global","2026-10-21","https://www.squarespace.com/robots.txt","200","ok","2379","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","7","has_strict_links","7","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/squarespace.com.txt","2280dba517d015842f2e97870bf9c17952ac25fec56b9685637501e839d8ca3b","2280dba517d015842f2e97870bf9c17952ac25fec56b9685637501e839d8ca3b","Matches source index","https://squarespace.com/robots.txt","https://www.squarespace.com/robots.txt","llms/squarespace.com.txt","b74a33b3ffd0177e8ae63a53467a54a249df355cff1c18c83d0b079751834f36","b74a33b3ffd0177e8ae63a53467a54a249df355cff1c18c83d0b079751834f36","Matches source index","https://squarespace.com/llms.txt","https://www.squarespace.com/assets/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/squarespace.com.txt","llms/squarespace.com.txt","2280dba517d015842f2e97870bf9c17952ac25fec56b9685637501e839d8ca3b","b74a33b3ffd0177e8ae63a53467a54a249df355cff1c18c83d0b079751834f36","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"110","docusign.com","DocuSign","SaaS & technology platforms","E-signature software","United States","Global","2026-10-21","https://www.docusign.com/robots.txt","200","ok","1931","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/docusign.com.txt","b645aa0a84da4faf77ef9e89069df9a7d16bcbeae8bc5a16993715cc7ab99020","b645aa0a84da4faf77ef9e89069df9a7d16bcbeae8bc5a16993715cc7ab99020","Matches source index","https://docusign.com/robots.txt","https://www.docusign.com/robots.txt","","","f00fb17ca1e84c27f0bc3550ac180afb3d083ee64a5d0b9c4e5b12222b81e7cc","No archived body","https://docusign.com/llms.txt","https://www.docusign.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/docusign.com.txt","","b645aa0a84da4faf77ef9e89069df9a7d16bcbeae8bc5a16993715cc7ab99020","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"111","okta.com","Okta","SaaS & technology platforms","Identity platform","United States","Global","2026-10-21","https://www.okta.com/robots.txt","200","ok","2518","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","1511","has_strict_links","1511","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/okta.com.txt","c1f9eb29d75800e04f12993a7c62259ed447a6afb5d4ee352318cd2469d5c533","c1f9eb29d75800e04f12993a7c62259ed447a6afb5d4ee352318cd2469d5c533","Matches source index","https://okta.com/robots.txt","https://www.okta.com/robots.txt","llms/okta.com.txt","51331ea8f4d68529aeb6d2273b87a50a60b11998379d4b82894975887f5279e9","51331ea8f4d68529aeb6d2273b87a50a60b11998379d4b82894975887f5279e9","Matches source index","https://okta.com/llms.txt","https://www.okta.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/okta.com.txt","llms/okta.com.txt","c1f9eb29d75800e04f12993a7c62259ed447a6afb5d4ee352318cd2469d5c533","51331ea8f4d68529aeb6d2273b87a50a60b11998379d4b82894975887f5279e9","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"112","crowdstrike.com","CrowdStrike","SaaS & technology platforms","Cybersecurity platform","United States","Global","2026-10-21","https://www.crowdstrike.com/robots.txt","200","ok","1822","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","16","has_strict_links","16","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/crowdstrike.com.txt","cddfd843f0d2606952ac1b5039c22a825d235827f0f0c277bca7bb3e724d5c14","cddfd843f0d2606952ac1b5039c22a825d235827f0f0c277bca7bb3e724d5c14","Matches source index","https://crowdstrike.com/robots.txt","https://www.crowdstrike.com/robots.txt","llms/crowdstrike.com.txt","624026d413e91f47418f35de796796844f218b6cfe91e70e355fc6d87ee58061","624026d413e91f47418f35de796796844f218b6cfe91e70e355fc6d87ee58061","Matches source index","https://crowdstrike.com/llms.txt","https://www.crowdstrike.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/crowdstrike.com.txt","llms/crowdstrike.com.txt","cddfd843f0d2606952ac1b5039c22a825d235827f0f0c277bca7bb3e724d5c14","624026d413e91f47418f35de796796844f218b6cfe91e70e355fc6d87ee58061","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"113","splunk.com","Splunk","SaaS & technology platforms","Data analytics platform","United States","Global","2026-10-21","https://www.splunk.com/robots.txt","200","ok","7689","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","splunk.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/splunk.com.txt","07ff6b38c1023e20a07245684f44a6b5cbe3bcd7faf2f8651742aba86b8c4539","07ff6b38c1023e20a07245684f44a6b5cbe3bcd7faf2f8651742aba86b8c4539","Matches source index","https://splunk.com/robots.txt","https://www.splunk.com/robots.txt","","","0023d4cdc49275e10c559833338c7adbd13dee33809366dfc198569674adc6cc","No archived body","https://splunk.com/llms.txt","https://www.splunk.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/splunk.com.txt","","07ff6b38c1023e20a07245684f44a6b5cbe3bcd7faf2f8651742aba86b8c4539","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"114","paloaltonetworks.com","Palo Alto Networks","SaaS & technology platforms","Cybersecurity platform","United States","Global","2026-10-21","https://www.paloaltonetworks.com/robots.txt","200","ok","7701","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","115","has_strict_links","115","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/paloaltonetworks.com.txt","329486a1c300341e2b2ad756655830d5be25f139a09a7d0a7425408b896ba157","329486a1c300341e2b2ad756655830d5be25f139a09a7d0a7425408b896ba157","Matches source index","https://paloaltonetworks.com/robots.txt","https://www.paloaltonetworks.com/robots.txt","llms/paloaltonetworks.com.txt","14e55414db825c4385a8e3ade91481b28fcc6392c1a43f67ae945eaf97f42bd6","14e55414db825c4385a8e3ade91481b28fcc6392c1a43f67ae945eaf97f42bd6","Matches source index","https://paloaltonetworks.com/llms.txt","https://www.paloaltonetworks.com/llms.txt","200","text/plain;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/paloaltonetworks.com.txt","llms/paloaltonetworks.com.txt","329486a1c300341e2b2ad756655830d5be25f139a09a7d0a7425408b896ba157","14e55414db825c4385a8e3ade91481b28fcc6392c1a43f67ae945eaf97f42bd6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"115","amd.com","AMD","SaaS & technology platforms","Semiconductors","United States","Global","2026-10-21","https://www.amd.com/robots.txt","200","ok","1300","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/amd.com.txt","0b1fd3de47d303a51102f2e720c1f447a3666b80cbf9f36bacc4a184cb186194","0b1fd3de47d303a51102f2e720c1f447a3666b80cbf9f36bacc4a184cb186194","Matches source index","https://amd.com/robots.txt","https://www.amd.com/robots.txt","","","9d32c91d8588a305f28302f22cce13264f39f5c1a57c00814caaf9aa65bd2001","No archived body","https://amd.com/llms.txt","https://www.amd.com/en/site-notifications/404.html?ref=/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/amd.com.txt","","0b1fd3de47d303a51102f2e720c1f447a3666b80cbf9f36bacc4a184cb186194","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"116","qualcomm.com","Qualcomm","SaaS & technology platforms","Semiconductors","United States","Global","2026-10-21","https://www.qualcomm.com/robots.txt","200","ok","492","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","6","has_strict_links","6","True","qualcomm.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/qualcomm.com.txt","f1867556a1802a43711fe15690c57b2c9f71c762ffab771fef2dedea7e992ce8","f1867556a1802a43711fe15690c57b2c9f71c762ffab771fef2dedea7e992ce8","Matches source index","https://qualcomm.com/robots.txt","https://www.qualcomm.com/robots.txt","llms/qualcomm.com.txt","4ab9d174698927cfb9f4fb18d31f4fb44d50804accdd5b1ba29a13f954056d3e","4ab9d174698927cfb9f4fb18d31f4fb44d50804accdd5b1ba29a13f954056d3e","Matches source index","https://qualcomm.com/llms.txt","https://www.qualcomm.com/llms.txt","200","text/markdown; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/qualcomm.com.txt","llms/qualcomm.com.txt","f1867556a1802a43711fe15690c57b2c9f71c762ffab771fef2dedea7e992ce8","4ab9d174698927cfb9f4fb18d31f4fb44d50804accdd5b1ba29a13f954056d3e","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"117","broadcom.com","Broadcom","SaaS & technology platforms","Semiconductors","United States","Global","2026-10-21","https://www.broadcom.com/robots.txt","200","ok","1366","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/broadcom.com.txt","9a5909af96840ea21725326bb3c54c8fc54d6d2228801e5afd27345238b883d2","9a5909af96840ea21725326bb3c54c8fc54d6d2228801e5afd27345238b883d2","Matches source index","https://broadcom.com/robots.txt","https://www.broadcom.com/robots.txt","","","51432bf2c10bd5d901df8a570a103ac34131c8e8e96515a8d90ce916c81d53c3","No archived body","https://broadcom.com/llms.txt","https://www.broadcom.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/broadcom.com.txt","","9a5909af96840ea21725326bb3c54c8fc54d6d2228801e5afd27345238b883d2","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"118","huawei.com","Huawei","SaaS & technology platforms","Telecom equipment & consumer technology","China","Global","2026-10-22","https://www.huawei.com/robots.txt","200","ok","2998","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Max retries exceeded with url: /","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/huawei.com.txt","83cf801b611039e70f80f9b8228d43fa91eb316e1990e6eb59e385d37e2d308a","83cf801b611039e70f80f9b8228d43fa91eb316e1990e6eb59e385d37e2d308a","Matches source index","https://huawei.com/robots.txt","https://www.huawei.com/robots.txt","","","ebd6297c0d8b6ed5491e9dc72c969d885cc42259d6be47cc5ae41a0a3aef6a1c","No archived body","https://www.huawei.com/llms.txt","https://www.huawei.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/huawei.com.txt","","83cf801b611039e70f80f9b8228d43fa91eb316e1990e6eb59e385d37e2d308a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"119","ericsson.com","Ericsson","SaaS & technology platforms","Telecom equipment","Sweden","Global","2026-10-21","https://ericsson.com/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://ericsson.com/robots.txt","https://ericsson.com/robots.txt","","","a980b60a8922f510d2da527e74ec9443a57dcc65444dbd6a3ae87dceb28090eb","No archived body","https://ericsson.com/llms.txt","https://ericsson.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"120","jetbrains.com","JetBrains","SaaS & technology platforms","Developer tools","Czechia","Global","2026-10-21","https://www.jetbrains.com/robots.txt","200","ok","1760","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jetbrains.com.txt","9372e147d5d59b2578c0753fd07e7567f6db8f12b2aa3604248f7d7b13a480e9","9372e147d5d59b2578c0753fd07e7567f6db8f12b2aa3604248f7d7b13a480e9","Matches source index","https://jetbrains.com/robots.txt","https://www.jetbrains.com/robots.txt","","","70e7edc57b0853ad2ea07734fba576e825e0ab094d90950b3d4cc541021cdba0","No archived body","https://jetbrains.com/llms.txt","https://www.jetbrains.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jetbrains.com.txt","","9372e147d5d59b2578c0753fd07e7567f6db8f12b2aa3604248f7d7b13a480e9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"121","amazon.com","Amazon","E-commerce & retail","Marketplace","United States","Global","2026-10-21","https://www.amazon.com/robots.txt","200","ok","7947","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/amazon.com.txt","74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","Matches source index","https://amazon.com/robots.txt","https://www.amazon.com/robots.txt","","","34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","No archived body","https://amazon.com/llms.txt","https://www.amazon.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/amazon.com.txt","","74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"122","ebay.com","eBay","E-commerce & retail","Marketplace","United States","Global","2026-10-21","https://www.ebay.com/robots.txt","200","ok","24164","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ebay.com.txt","221797cd7a314aeba19bb83d7868af3ea04325e4f2f8fd9b8ba465649e091970","221797cd7a314aeba19bb83d7868af3ea04325e4f2f8fd9b8ba465649e091970","Matches source index","https://ebay.com/robots.txt","https://www.ebay.com/robots.txt","","","321b979378a5d41790fd4379294a337b0e8573483a91585e516e3fe4b2c459f6","No archived body","https://ebay.com/llms.txt","https://pages.ebay.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ebay.com.txt","","221797cd7a314aeba19bb83d7868af3ea04325e4f2f8fd9b8ba465649e091970","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"123","walmart.com","Walmart","E-commerce & retail","Retail","United States","Global","2026-10-21","https://www.walmart.com/robots.txt","200","ok","3479","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/walmart.com.txt","502a5efeb6a8672295f44c12387975533e9441f931cc7fb9d3371ad18f5d9b34","502a5efeb6a8672295f44c12387975533e9441f931cc7fb9d3371ad18f5d9b34","Matches source index","https://walmart.com/robots.txt","https://www.walmart.com/robots.txt","","","a2c0f5cfb251f6d5a4630f965445a8eb7c8d26d730f99f04e3f5638dba419180","No archived body","https://walmart.com/llms.txt","https://www.walmart.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/walmart.com.txt","","502a5efeb6a8672295f44c12387975533e9441f931cc7fb9d3371ad18f5d9b34","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"124","target.com","Target","E-commerce & retail","Retail","United States","Global","2026-10-21","https://www.target.com/robots.txt","200","ok","3226","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","54","has_strict_links","54","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/target.com.txt","3f8a9e45a59d43ab1b0e4534ae71241fd9e6cbbd3c37b5426bc9efd6eb320bd7","3f8a9e45a59d43ab1b0e4534ae71241fd9e6cbbd3c37b5426bc9efd6eb320bd7","Matches source index","https://target.com/robots.txt","https://www.target.com/robots.txt","llms/target.com.txt","401b6c28fa673c7993916e8a6ce4ac33e67aba61e27b8f0cd67ecb7b09d02c4d","401b6c28fa673c7993916e8a6ce4ac33e67aba61e27b8f0cd67ecb7b09d02c4d","Matches source index","https://target.com/llms.txt","https://www.target.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/target.com.txt","llms/target.com.txt","3f8a9e45a59d43ab1b0e4534ae71241fd9e6cbbd3c37b5426bc9efd6eb320bd7","401b6c28fa673c7993916e8a6ce4ac33e67aba61e27b8f0cd67ecb7b09d02c4d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"125","etsy.com","Etsy","E-commerce & retail","Marketplace","United States","Global","2026-10-21","https://www.etsy.com/robots.txt","200","ok","52811","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","76","has_strict_links","81","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/etsy.com.txt","6ebe2465427dc327c1e067e07b96eee9cc6b69cd2a5395995c9cdc45f573d188","6ebe2465427dc327c1e067e07b96eee9cc6b69cd2a5395995c9cdc45f573d188","Matches source index","https://etsy.com/robots.txt","https://www.etsy.com/robots.txt","llms/etsy.com.txt","d1f3a9aec5f2174cd3500aeff468ed31bdd0a114adee97c28357aaac64580c2e","d1f3a9aec5f2174cd3500aeff468ed31bdd0a114adee97c28357aaac64580c2e","Matches source index","https://etsy.com/llms.txt","https://www.etsy.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/etsy.com.txt","llms/etsy.com.txt","6ebe2465427dc327c1e067e07b96eee9cc6b69cd2a5395995c9cdc45f573d188","d1f3a9aec5f2174cd3500aeff468ed31bdd0a114adee97c28357aaac64580c2e","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"126","bestbuy.com","Best Buy","E-commerce & retail","Electronics retail","United States","Global","2026-10-22","https://www.bestbuy.com/robots.txt","200","ok","15016","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","bestbuy.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bestbuy.com.txt","cb65f883b9aeec87339db0e12aa3228f8b8b49613f0beeb58111bf3a153e8de9","cb65f883b9aeec87339db0e12aa3228f8b8b49613f0beeb58111bf3a153e8de9","Matches source index","https://bestbuy.com/robots.txt","https://www.bestbuy.com/robots.txt","","","7a3efca3eb18d7b6facd25b01bed61fdaba72e79a66c4626af855a8d701dbaa2","No archived body","https://bestbuy.com/llms.txt","https://www.bestbuy.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/bestbuy.com.txt","","cb65f883b9aeec87339db0e12aa3228f8b8b49613f0beeb58111bf3a153e8de9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"127","homedepot.com","The Home Depot","E-commerce & retail","Home improvement retail","United States","Global","2026-10-21","https://www.homedepot.com/robots.txt","200","ok","1394","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/homedepot.com.txt","74c50c533fb5af97bf237e6ff65f94823fb90d9b5447b2e40a55166215e0c232","74c50c533fb5af97bf237e6ff65f94823fb90d9b5447b2e40a55166215e0c232","Matches source index","https://homedepot.com/robots.txt","https://www.homedepot.com/robots.txt","","","3d342a05729e2395aa0e47f62fb04c5d415c73449f0c051f4348684ecde55adc","No archived body","https://homedepot.com/llms.txt","https://www.homedepot.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/homedepot.com.txt","","74c50c533fb5af97bf237e6ff65f94823fb90d9b5447b2e40a55166215e0c232","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-d77cebb96001","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"128","lowes.com","Lowe's","E-commerce & retail","Home improvement retail","United States","Global","2026-10-22","https://lowes.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://lowes.com/robots.txt","https://lowes.com/robots.txt","","","d240e1b73d1d5db534502579a95eeb4023667531bc4af5578bdb89f95a31ffac","No archived body","https://lowes.com/llms.txt","https://www.lowes.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-566e496bf87a","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","500","10222353-2533-0275-0000-0ca3e5f9af17","2026-10-22 23:54:14 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"129","costco.com","Costco","E-commerce & retail","Warehouse retail","United States","Global","2026-10-21","https://www.costco.com/robots.txt","200","ok","3997","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/costco.com.txt","ed8ed1579bf2033e61209616014293912eb9770ec86c4a49d54c79c8b6efcb27","ed8ed1579bf2033e61209616014293912eb9770ec86c4a49d54c79c8b6efcb27","Matches source index","https://costco.com/robots.txt","https://www.costco.com/robots.txt","","","7b6ab9587085c95a9df8fd9c7363791f8f6936f94f88f553efe5b9a91a4620b3","No archived body","https://costco.com/llms.txt","https://www.costco.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/costco.com.txt","","ed8ed1579bf2033e61209616014293912eb9770ec86c4a49d54c79c8b6efcb27","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"130","wayfair.com","Wayfair","E-commerce & retail","Furniture retail","United States","Global","2026-10-21","https://www.wayfair.com/robots.txt","200","ok","3224","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wayfair.com.txt","c00436f6bfb9eef80d98fc78c591c15a452d58ca20e982e25fa05a92eb74ae3a","c00436f6bfb9eef80d98fc78c591c15a452d58ca20e982e25fa05a92eb74ae3a","Matches source index","https://wayfair.com/robots.txt","https://www.wayfair.com/robots.txt","","","14e067e55f824e0e4121e4d9412fb9b1cc733225a33959fac8f0e507ade8b139","No archived body","https://wayfair.com/llms.txt","https://www.wayfair.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/wayfair.com.txt","","c00436f6bfb9eef80d98fc78c591c15a452d58ca20e982e25fa05a92eb74ae3a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"131","chewy.com","Chewy","E-commerce & retail","Pet retail","United States","Global","2026-10-21","https://www.chewy.com/robots.txt","200","ok","2513","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/chewy.com.txt","222b0261ee4d989851592e7ede082076354b032c04bce9998052ba7cce54632f","222b0261ee4d989851592e7ede082076354b032c04bce9998052ba7cce54632f","Matches source index","https://chewy.com/robots.txt","https://www.chewy.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://chewy.com/llms.txt","https://www.chewy.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/chewy.com.txt","","222b0261ee4d989851592e7ede082076354b032c04bce9998052ba7cce54632f","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"132","kroger.com","Kroger","E-commerce & retail","Grocery retail","United States","Global","2026-10-22","https://kroger.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.kroger.com', port=80): Read timed out. (read timeout=20) kroger.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://kroger.com/robots.txt","https://kroger.com/robots.txt","","","","No archived body","https://kroger.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222349-2533-0275-0000-1fcce20f4aaa","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","","10222353-2533-0275-0000-62b22d758e4a","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"133","walgreens.com","Walgreens","E-commerce & retail","Pharmacy retail","United States","Global","2026-10-21","https://www.walgreens.com/robots.txt","200","ok","2593","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/walgreens.com.txt","f3b301ce426bc203f1fd60f2594383f743c766ee09b329e93614cff693dee835","f3b301ce426bc203f1fd60f2594383f743c766ee09b329e93614cff693dee835","Matches source index","https://walgreens.com/robots.txt","https://www.walgreens.com/robots.txt","llms/walgreens.com.txt","1b9c175d26bf53957da909292226eaa9ac320daf81d13066f6de8b65069297a0","1b9c175d26bf53957da909292226eaa9ac320daf81d13066f6de8b65069297a0","Matches source index","https://www.walgreens.com/llms.txt","https://www.walgreens.com/request_error.jsp?redirect=/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/walgreens.com.txt","llms/walgreens.com.txt","f3b301ce426bc203f1fd60f2594383f743c766ee09b329e93614cff693dee835","1b9c175d26bf53957da909292226eaa9ac320daf81d13066f6de8b65069297a0","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"134","cvs.com","CVS Pharmacy","E-commerce & retail","Pharmacy retail","United States","Global","2026-10-21","https://www.cvs.com/robots.txt","200","ok","7836","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present_nonstandard_format","nonstandard_site_metadata","0","no_strict_links","60","True","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Present, nonstandard format","v3.2-verification","November 2026","raw/cvs.com.txt","59f38b15d0eb77d55dc5969a113d83d94a731a8705f97b3cab8fee8f90093071","59f38b15d0eb77d55dc5969a113d83d94a731a8705f97b3cab8fee8f90093071","Matches source index","https://cvs.com/robots.txt","https://www.cvs.com/robots.txt","crawler-api-verification-pass/raw/llms/cvs.com--10222353-2533-0275-0000-019261363099.txt","a0d5fff7cb15bbd697e48c24ed59e9bd497f468e36786172ba3c371025935c76","","API-returned UTF-8 body; byte length matched provider size","https://cvs.com/llms.txt","https://www.cvs.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/cvs.com.txt","","59f38b15d0eb77d55dc5969a113d83d94a731a8705f97b3cab8fee8f90093071","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present_nonstandard_format","200","10222353-2533-0275-0000-019261363099","2026-10-22 23:53:54 +00:00","Content-based site-reference document with Site/Owner/Purpose metadata and multiple URLs; excluded from the pinned first-heading/strict-link detection rule, not absent.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"135","macys.com","Macy's","E-commerce & retail","Department store","United States","Global","2026-10-22","https://macys.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://macys.com/robots.txt","https://macys.com/robots.txt","","","a0014cc98f0dbab131fc54c18140a91e82daedc2f7470e45c165e36e392f75a2","No archived body","https://macys.com/llms.txt","https://www.macys.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-0366b7b7cf50","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-5a67f24c7f7d","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"136","nordstrom.com","Nordstrom","E-commerce & retail","Department store","United States","Global","2026-10-21","https://www.nordstrom.com/robots.txt","200","ok","2685","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nordstrom.com.txt","73fbcd4795943ac2fa529794c99544f10bc3573634f50ca3a4c540056b0486e5","73fbcd4795943ac2fa529794c99544f10bc3573634f50ca3a4c540056b0486e5","Matches source index","https://nordstrom.com/robots.txt","https://www.nordstrom.com/robots.txt","","","7fca2b5e7f477891e0c06251641650f0f63d98c5a02d86ae20c76d2920328a8f","No archived body","https://nordstrom.com/llms.txt","https://www.nordstrom.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nordstrom.com.txt","","73fbcd4795943ac2fa529794c99544f10bc3573634f50ca3a4c540056b0486e5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"137","nike.com","Nike","E-commerce & retail","Sportswear retail","United States","Global","2026-10-21","https://www.nike.com/robots.txt","200","ok","3503","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nike.com.txt","54ba534bff9e8b4cf89f30c9432c8aa4b8e2b652cb1a9170b25ac8d8ad356ee6","54ba534bff9e8b4cf89f30c9432c8aa4b8e2b652cb1a9170b25ac8d8ad356ee6","Matches source index","https://nike.com/robots.txt","https://www.nike.com/robots.txt","","","9c146f03966a91dfca42081136c103413da81854334fab3d807b0e7d44b8bfdd","No archived body","https://nike.com/llms.txt","https://www.nike.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nike.com.txt","","54ba534bff9e8b4cf89f30c9432c8aa4b8e2b652cb1a9170b25ac8d8ad356ee6","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"138","adidas.com","Adidas","E-commerce & retail","Sportswear retail","Germany","Global","2026-10-22","https://adidas.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://adidas.com/robots.txt","https://adidas.com/robots.txt","","","5ec3b27148b0624de9eac602ed4f0aa62277306daa60abce0a86a018f8dce1b5","No archived body","https://adidas.com/llms.txt","https://www.adidas.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-69d774870560","2026-10-22 23:49:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-f294887447c6","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"139","ikea.com","IKEA","E-commerce & retail","Furniture retail","Netherlands","Global","2026-10-21","https://www.ikea.com/robots.txt","200","ok","4054","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ikea.com.txt","53df86f20aea5f91125e8ed0d4ae2c69aaf94fd671a276b41354a5a62b4a9338","53df86f20aea5f91125e8ed0d4ae2c69aaf94fd671a276b41354a5a62b4a9338","Matches source index","https://ikea.com/robots.txt","https://www.ikea.com/robots.txt","","","0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","No archived body","https://ikea.com/llms.txt","https://www.ikea.com/llms.txt","404","text/plain; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ikea.com.txt","","53df86f20aea5f91125e8ed0d4ae2c69aaf94fd671a276b41354a5a62b4a9338","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"140","alibaba.com","Alibaba","E-commerce & retail","Marketplace","China","Global","2026-10-21","https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt","200","ok","5406","valid_txt_200","True","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/alibaba.com.txt","a4a98eee7f6330d376232fa8eaef77ab924702e2fdd91e071b4f2c27501b8cae","a4a98eee7f6330d376232fa8eaef77ab924702e2fdd91e071b4f2c27501b8cae","Matches source index","https://alibaba.com/robots.txt","https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt","llms/alibaba.com.txt","2aaf78d7ca5f71e4caa0f1f64a11c26fd4753c9c7d1c3d772f82e3ab85222a85","2aaf78d7ca5f71e4caa0f1f64a11c26fd4753c9c7d1c3d772f82e3ab85222a85","Matches source index","https://alibaba.com/llms.txt","https://error.alibaba.com/error404.htm","200","text/html;charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/alibaba.com.txt","llms/alibaba.com.txt","a4a98eee7f6330d376232fa8eaef77ab924702e2fdd91e071b4f2c27501b8cae","2aaf78d7ca5f71e4caa0f1f64a11c26fd4753c9c7d1c3d772f82e3ab85222a85","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"141","jd.com","JD.com","E-commerce & retail","Marketplace","China","Global","2026-10-21","https://jd.com/robots.txt","200","ok","786","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jd.com.txt","3e033ad639e1be74865f620abd440088a90fd889f9c55584ecaf4a166987be1b","3e033ad639e1be74865f620abd440088a90fd889f9c55584ecaf4a166987be1b","Matches source index","https://jd.com/robots.txt","https://jd.com/robots.txt","llms/jd.com.txt","6f7a622691fabb5d05d714856354c8caee9916d96ebbd22adc6afc4e58b3a389","6f7a622691fabb5d05d714856354c8caee9916d96ebbd22adc6afc4e58b3a389","Matches source index","https://jd.com/llms.txt","https://global.jd.com","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jd.com.txt","llms/jd.com.txt","3e033ad639e1be74865f620abd440088a90fd889f9c55584ecaf4a166987be1b","6f7a622691fabb5d05d714856354c8caee9916d96ebbd22adc6afc4e58b3a389","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"142","aliexpress.com","AliExpress","E-commerce & retail","Marketplace","China","Global","2026-10-21","https://www.aliexpress.com/robots.txt","200","ok","3025","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aliexpress.com.txt","8b70ec63378682631ceaa01e5892d657aafeae1e0f14fa653093d38460d53097","8b70ec63378682631ceaa01e5892d657aafeae1e0f14fa653093d38460d53097","Matches source index","https://aliexpress.com/robots.txt","https://www.aliexpress.com/robots.txt","llms/aliexpress.com.txt","e66c424fe5d31c8655d546d0f78c177ce496ad4f9884833a4c2c4c3d4cff32fa","e66c424fe5d31c8655d546d0f78c177ce496ad4f9884833a4c2c4c3d4cff32fa","Matches source index","https://aliexpress.com/llms.txt","https://www.aliexpress.com/s/error/404?wh_pid=%2Fae%2Fmega%2Fae%2F2021_global_shopping_festival%2F2021topbrands_for_you&wh_weex=true&af=31746480&dp=71e4abab7fe4e6eb10ad22fac505ca9e&aff_fcid=2b84f1c089c245d880e0d36f6f5c19d5-1792675756325-04763-_AESytN&tt=CPS_NORMAL&aff_fsk=_AESytN&aff_platform=portals-promotion&sk=_AESytN&aff_trace_key=2b84f1c089c245d880e0d36f6f5c19d5-1792675756325-04763-_AESytN&terminal_id=ecec82c0f2814491826e2a550a462e1b","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aliexpress.com.txt","llms/aliexpress.com.txt","8b70ec63378682631ceaa01e5892d657aafeae1e0f14fa653093d38460d53097","e66c424fe5d31c8655d546d0f78c177ce496ad4f9884833a4c2c4c3d4cff32fa","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"143","flipkart.com","Flipkart","E-commerce & retail","Marketplace","India","Global","2026-10-22","https://www.flipkart.com/robots.txt","200","ok","3111","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/flipkart.com.txt","e86fb1833717792a9f0a1588a2c73b6f1561b88f6828c132da20aa7fdc8173a8","e86fb1833717792a9f0a1588a2c73b6f1561b88f6828c132da20aa7fdc8173a8","Matches source index","https://flipkart.com/robots.txt","https://www.flipkart.com/robots.txt","","","0ef2843ac78a5e876c4bbf4687e2910811d521d05935f415c7e08ac4baa2fbe1","No archived body","https://flipkart.com/llms.txt","https://www.flipkart.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/flipkart.com.txt","","e86fb1833717792a9f0a1588a2c73b6f1561b88f6828c132da20aa7fdc8173a8","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"144","mercadolibre.com","Mercado Libre","E-commerce & retail","Marketplace","Argentina","Global","2026-10-21","https://mercadolibre.com/robots.txt","200","ok","474","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mercadolibre.com.txt","ee5f9fc0dc4f0f40286e295d3972622a092f7663baa88ffb214b5254af41f51d","ee5f9fc0dc4f0f40286e295d3972622a092f7663baa88ffb214b5254af41f51d","Matches source index","https://mercadolibre.com/robots.txt","https://mercadolibre.com/robots.txt","","","b7656941ce60b3e409a6048919a3bb6a31dc8cc8793ccef3d5313e4c1eb97ed5","No archived body","https://mercadolibre.com/llms.txt","https://mercadolibre.com/llms.txt","404","application/json; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mercadolibre.com.txt","","ee5f9fc0dc4f0f40286e295d3972622a092f7663baa88ffb214b5254af41f51d","","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"145","rakuten.com","Rakuten","E-commerce & retail","Marketplace","Japan","Global","2026-10-21","https://www.rakuten.com/robots.txt","200","ok","2621","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/rakuten.com.txt","8bccbc078da0f012742040175fe4dfabddedf46303e466ff0ec112aa8eeaab56","8bccbc078da0f012742040175fe4dfabddedf46303e466ff0ec112aa8eeaab56","Matches source index","https://rakuten.com/robots.txt","https://www.rakuten.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://rakuten.com/llms.txt","http://www.rakuten.com/404","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/rakuten.com.txt","","8bccbc078da0f012742040175fe4dfabddedf46303e466ff0ec112aa8eeaab56","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"146","zalando.de","Zalando","E-commerce & retail","Fashion retail","Germany","Global","2026-10-22","https://www.zalando.de/robots.txt","200","ok","644","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","104","has_strict_links","104","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/zalando.de.txt","470ca4815732e8aa531535cffbf1a7cfbf1a250f8b0fe90418f547ee101c244f","470ca4815732e8aa531535cffbf1a7cfbf1a250f8b0fe90418f547ee101c244f","Matches source index","https://zalando.de/robots.txt","https://www.zalando.de/robots.txt","llms/zalando.de.txt","2cdd6d607223ec93cbc4351b1ac49afa3fa9ee273f3acf4587bc0a54f074159c","2cdd6d607223ec93cbc4351b1ac49afa3fa9ee273f3acf4587bc0a54f074159c","Matches source index","https://zalando.de/llms.txt","https://www.zalando.de/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/zalando.de.txt","llms/zalando.de.txt","470ca4815732e8aa531535cffbf1a7cfbf1a250f8b0fe90418f547ee101c244f","2cdd6d607223ec93cbc4351b1ac49afa3fa9ee273f3acf4587bc0a54f074159c","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"147","asos.com","ASOS","E-commerce & retail","Fashion retail","United Kingdom","Global","2026-10-22","https://asos.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, Non asos.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://asos.com/robots.txt","https://asos.com/robots.txt","","","777e60ae61e13bc2d669b1125faea988462b9505911da11018a7c1f017f30435","No archived body","https://asos.com/llms.txt","https://www.asos.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222349-2533-0275-0000-ef478f9abc33","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-5a19d75fd5d0","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"148","shopify.com","Shopify","E-commerce & retail","E-commerce platform","Canada","Global","2026-10-21","https://shopify.com/robots.txt","200","ok","62","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","74","has_strict_links","75","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/shopify.com.txt","a55c031afe721803d5e722c76910b8b569511cce2fc45459933c60d8e0764aa7","a55c031afe721803d5e722c76910b8b569511cce2fc45459933c60d8e0764aa7","Matches source index","https://shopify.com/robots.txt","https://shopify.com/robots.txt","llms/shopify.com.txt","fd5d372a21847ff2333682645367699f10c44ded4a1dbf08d8e93d512beb765a","fd5d372a21847ff2333682645367699f10c44ded4a1dbf08d8e93d512beb765a","Matches source index","https://shopify.com/llms.txt","https://www.shopify.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/shopify.com.txt","llms/shopify.com.txt","a55c031afe721803d5e722c76910b8b569511cce2fc45459933c60d8e0764aa7","fd5d372a21847ff2333682645367699f10c44ded4a1dbf08d8e93d512beb765a","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"149","shein.com","Shein","E-commerce & retail","Fashion retail","","Global","2026-10-21","https://www.shein.com/robots.txt","200","ok","279","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","190","has_strict_links","196","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/shein.com.txt","7a6d32e222e319a6425d08100b814b1ab110ad21d1b08b520faeaa158d973735","7a6d32e222e319a6425d08100b814b1ab110ad21d1b08b520faeaa158d973735","Matches source index","https://shein.com/robots.txt","https://www.shein.com/robots.txt","llms/shein.com.txt","19f2eb866e9b8eff8ceb9cee0eca9dd614a9ab7dbf31b77b205a8be876c0c90d","19f2eb866e9b8eff8ceb9cee0eca9dd614a9ab7dbf31b77b205a8be876c0c90d","Matches source index","https://shein.com/llms.txt","https://www.shein.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/shein.com.txt","llms/shein.com.txt","7a6d32e222e319a6425d08100b814b1ab110ad21d1b08b520faeaa158d973735","19f2eb866e9b8eff8ceb9cee0eca9dd614a9ab7dbf31b77b205a8be876c0c90d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"150","temu.com","Temu","E-commerce & retail","Marketplace","","Global","2026-10-21","https://www.temu.com/robots.txt","200","ok","8239","valid_txt_200","True","Partial","Partial","Allowed","Partial","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/temu.com.txt","e6791399506405731bf4f737044195aee793aa81d5c17e12f7a79d82c0b3d8b8","e6791399506405731bf4f737044195aee793aa81d5c17e12f7a79d82c0b3d8b8","Matches source index","https://temu.com/robots.txt","https://www.temu.com/robots.txt","llms/temu.com.txt","c520d4db375894003b6a65c169f307433a494a064dcb8e16507d8f3a15bfc399","c520d4db375894003b6a65c169f307433a494a064dcb8e16507d8f3a15bfc399","Matches source index","https://temu.com/llms.txt","https://www.temu.com/","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/temu.com.txt","llms/temu.com.txt","e6791399506405731bf4f737044195aee793aa81d5c17e12f7a79d82c0b3d8b8","c520d4db375894003b6a65c169f307433a494a064dcb8e16507d8f3a15bfc399","Partial","Partial","Allowed","Partial","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"151","wish.com","Wish","E-commerce & retail","Marketplace","United States","Global","2026-10-21","https://www.wish.com/robots.txt","200","ok","565","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wish.com.txt","1485dd15bf8187f48a4afed18920d986e3ac1a2c541ab41e1a77dde9f1288335","1485dd15bf8187f48a4afed18920d986e3ac1a2c541ab41e1a77dde9f1288335","Matches source index","https://wish.com/robots.txt","https://www.wish.com/robots.txt","","","3549f22f8907f02a9bdab45a27dedc4de6021f302f385111565b6d0a26885e67","No archived body","https://wish.com/llms.txt","https://www.wish.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/wish.com.txt","","1485dd15bf8187f48a4afed18920d986e3ac1a2c541ab41e1a77dde9f1288335","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"152","overstock.com","Overstock","E-commerce & retail","Furniture retail","United States","Global","2026-10-21","https://www.overstock.com/robots.txt","200","ok","1282","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/overstock.com.txt","164ee46876d5047dbcaa26f8ec6c61f4c1142216278be6263b9ce0b4bbdcd538","164ee46876d5047dbcaa26f8ec6c61f4c1142216278be6263b9ce0b4bbdcd538","Matches source index","https://overstock.com/robots.txt","https://www.overstock.com/robots.txt","","","44644d29e240a753213f7a6d90df0978bba4c5a649870263bc1891c04d1f06fc","No archived body","https://overstock.com/llms.txt","https://www.overstock.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/overstock.com.txt","","164ee46876d5047dbcaa26f8ec6c61f4c1142216278be6263b9ce0b4bbdcd538","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"153","zappos.com","Zappos","E-commerce & retail","Footwear retail","United States","Global","2026-10-21","https://www.zappos.com/robots.txt","200","ok","1209","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/zappos.com.txt","209c679d9e6f9906c57c9b5e064a5b5d509f0325d2e0a7c1611bd661f0cb978f","209c679d9e6f9906c57c9b5e064a5b5d509f0325d2e0a7c1611bd661f0cb978f","Matches source index","https://zappos.com/robots.txt","https://www.zappos.com/robots.txt","","","d0e069db4c9c5f7fb2bc6aa507e4271f1ce9dc232e36c85f3010de83220ab7a4","No archived body","https://zappos.com/llms.txt","https://www.zappos.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/zappos.com.txt","","209c679d9e6f9906c57c9b5e064a5b5d509f0325d2e0a7c1611bd661f0cb978f","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"154","sephora.com","Sephora","E-commerce & retail","Beauty retail","France","Global","2026-10-21","https://www.sephora.com/robots.txt","200","ok","922","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","49","has_strict_links","49","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/sephora.com.txt","6db6d384d147400f69d1e18cb55c710eef6ae7abc0a2684f8017de7e8550ad70","6db6d384d147400f69d1e18cb55c710eef6ae7abc0a2684f8017de7e8550ad70","Matches source index","https://sephora.com/robots.txt","https://www.sephora.com/robots.txt","llms/sephora.com.txt","8dba879c99b1d8d4118a1434530888d3c75e5d5bc0c858e19f1c38abe2373ca7","8dba879c99b1d8d4118a1434530888d3c75e5d5bc0c858e19f1c38abe2373ca7","Matches source index","https://sephora.com/llms.txt","https://www.sephora.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/sephora.com.txt","llms/sephora.com.txt","6db6d384d147400f69d1e18cb55c710eef6ae7abc0a2684f8017de7e8550ad70","8dba879c99b1d8d4118a1434530888d3c75e5d5bc0c858e19f1c38abe2373ca7","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"155","ulta.com","Ulta Beauty","E-commerce & retail","Beauty retail","United States","Global","2026-10-21","https://www.ulta.com/robots.txt","200","ok","2483","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ulta.com.txt","14fd51cdc736b79855f8a1bc620707123b1e0a1ff6d704e87e025618f48b9c04","14fd51cdc736b79855f8a1bc620707123b1e0a1ff6d704e87e025618f48b9c04","Matches source index","https://ulta.com/robots.txt","https://www.ulta.com/robots.txt","llms/ulta.com.txt","0dce1aa7176212d0008e0a3470d8e5680da1add2027a66e594d0cc969f504817","0dce1aa7176212d0008e0a3470d8e5680da1add2027a66e594d0cc969f504817","Matches source index","https://ulta.com/llms.txt","https://www.ulta.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ulta.com.txt","llms/ulta.com.txt","14fd51cdc736b79855f8a1bc620707123b1e0a1ff6d704e87e025618f48b9c04","0dce1aa7176212d0008e0a3470d8e5680da1add2027a66e594d0cc969f504817","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"156","gap.com","Gap","E-commerce & retail","Fashion retail","United States","Global","2026-10-21","https://www.gap.com/robots.txt","200","ok","871","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gap.com.txt","1ddac2a1ff79f194fd8b8e0328376509b5e140e45e7bf5ac4910af82aaa69e93","1ddac2a1ff79f194fd8b8e0328376509b5e140e45e7bf5ac4910af82aaa69e93","Matches source index","https://gap.com/robots.txt","https://www.gap.com/robots.txt","","","27e6b0f28e5540f79842db1c1752336aedae42edd8bee552f4ed957d6319a44e","No archived body","https://gap.com/llms.txt","https://www.gap.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gap.com.txt","","1ddac2a1ff79f194fd8b8e0328376509b5e140e45e7bf5ac4910af82aaa69e93","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"157","hm.com","H&M","E-commerce & retail","Fashion retail","Sweden","Global","2026-10-22","https://hm.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://hm.com/robots.txt","https://hm.com/robots.txt","","","12762ed47f1535438b4875f928f5fb741d53cb3e8a95a2b497b15a8c657166fb","No archived body","https://hm.com/llms.txt","https://hm.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-eb04f879581e","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-0f8418aa88c1","2026-10-22 23:53:53 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"158","zara.com","Zara","E-commerce & retail","Fashion retail","Spain","Global","2026-10-21","https://www.zara.com/robots.txt","200","ok","1612","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/zara.com.txt","c3c56bd3e26c0780e327341e6e40765e7ce945e6d7d24e79bcf95f2b74b80e2d","c3c56bd3e26c0780e327341e6e40765e7ce945e6d7d24e79bcf95f2b74b80e2d","Matches source index","https://zara.com/robots.txt","https://www.zara.com/robots.txt","","","83392a8060f1e24f9bb44b66f9462ac36454e82edb4d2fe73d970a006e42d6f6","No archived body","https://zara.com/llms.txt","https://www.zara.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/zara.com.txt","","c3c56bd3e26c0780e327341e6e40765e7ce945e6d7d24e79bcf95f2b74b80e2d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"159","uniqlo.com","Uniqlo","E-commerce & retail","Fashion retail","Japan","Global","2026-10-21","https://www.uniqlo.com/robots.txt","200","ok","14095","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/uniqlo.com.txt","a6b8ac18ee34dcb69c9d4616e8730f3521f0922398a5413f00eb614e9bae2d37","a6b8ac18ee34dcb69c9d4616e8730f3521f0922398a5413f00eb614e9bae2d37","Matches source index","https://uniqlo.com/robots.txt","https://www.uniqlo.com/robots.txt","","","28dbc9de0445b1c404e9a14043e292dceda19e777ab94ba2c67b285b90b9f71d","No archived body","https://uniqlo.com/llms.txt","https://www.uniqlo.com/llms.txt","403","application/xml","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/uniqlo.com.txt","","a6b8ac18ee34dcb69c9d4616e8730f3521f0922398a5413f00eb614e9bae2d37","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-7cc158b9a4a2","2026-10-22 23:53:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"160","lego.com","LEGO","E-commerce & retail","Toy retail","Denmark","Global","2026-10-21","https://www.lego.com/robots.txt","200","ok","784","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/lego.com.txt","e6ed4b365a86e8bb081187911e9fc85ba46f7d9bee22a4a2949d619b69694949","e6ed4b365a86e8bb081187911e9fc85ba46f7d9bee22a4a2949d619b69694949","Matches source index","https://lego.com/robots.txt","https://www.lego.com/robots.txt","","","1444216d4f966a5b50f6cbf03106d403962d5832264569b56c59d623e11fe358","No archived body","https://lego.com/llms.txt","https://www.lego.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/lego.com.txt","","e6ed4b365a86e8bb081187911e9fc85ba46f7d9bee22a4a2949d619b69694949","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"161","decathlon.com","Decathlon","E-commerce & retail","Sporting goods retail","France","Global","2026-10-21","https://www.decathlon.com/robots.txt","200","ok","3657","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","8","has_strict_links","19","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/decathlon.com.txt","db9bce2468fa1a38cf7279e42243f2a3b4c694ca9873d2676278b82649b99606","db9bce2468fa1a38cf7279e42243f2a3b4c694ca9873d2676278b82649b99606","Matches source index","https://decathlon.com/robots.txt","https://www.decathlon.com/robots.txt","llms/decathlon.com.txt","4d237eaae88ea26c8904ff025a14dc6e1a582c7172b8171b4322b9a14e437231","4d237eaae88ea26c8904ff025a14dc6e1a582c7172b8171b4322b9a14e437231","Matches source index","https://www.decathlon.com/llms.txt","https://www.decathlon.com/llms.txt","200","text/markdown; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/decathlon.com.txt","llms/decathlon.com.txt","db9bce2468fa1a38cf7279e42243f2a3b4c694ca9873d2676278b82649b99606","4d237eaae88ea26c8904ff025a14dc6e1a582c7172b8171b4322b9a14e437231","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"162","carrefour.com","Carrefour","E-commerce & retail","Grocery retail","France","Global","2026-10-21","https://www.carrefour.com/robots.txt","200","ok","1706","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/carrefour.com.txt","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","Matches source index","https://carrefour.com/robots.txt","https://www.carrefour.com/robots.txt","","","25fb23868ebf48348f9e438e00cb9b9d9b3a054f32482a781c762cc4f9cc6393","No archived body","https://carrefour.com/llms.txt","https://www.carrefour.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/carrefour.com.txt","","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222353-2533-0275-0000-d5ec99df9383","2026-10-22 23:53:54 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-01.json","2"
"163","tesco.com","Tesco","E-commerce & retail","Grocery retail","United Kingdom","Global","2026-10-22","https://tesco.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://tesco.com/robots.txt","https://tesco.com/robots.txt","","","f2596395a2fc076ed5ec577715dccc44bd79faf37f882d2d2f6846127c930329","No archived body","https://tesco.com/llms.txt","https://tesco.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-ca9180ce0f06","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-3cb89f6f19d0","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"164","sainsburys.co.uk","Sainsbury's","E-commerce & retail","Grocery retail","United Kingdom","Global","2026-10-22","https://sainsburys.co.uk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://sainsburys.co.uk/robots.txt","https://sainsburys.co.uk/robots.txt","","","3e777c7fa54cb8da6bed9f0c42b58b7bfda2016562810caaec4882c9eff6eea0","No archived body","https://sainsburys.co.uk/llms.txt","https://sainsburys.co.uk/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-07fdb4f67a2c","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-d6e2d998dc29","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"165","argos.co.uk","Argos","E-commerce & retail","Retail","United Kingdom","Global","2026-10-22","https://argos.co.uk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://argos.co.uk/robots.txt","https://argos.co.uk/robots.txt","","","3e65b1b28b23ee79d5325a5246da259dc9ca7ddc8cac94186d69bc107e2ad326","No archived body","https://argos.co.uk/llms.txt","https://argos.co.uk/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-3e35395f8fdc","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-25abdb79d33e","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"166","otto.de","Otto","E-commerce & retail","Marketplace","Germany","Global","2026-10-21","https://www.otto.de/robots.txt","200","ok","2148","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","102","has_strict_links","102","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/otto.de.txt","86abc355699826c5ac3f6b8c847f9889b7fa68fcc1176bad10a86892a8e99f85","86abc355699826c5ac3f6b8c847f9889b7fa68fcc1176bad10a86892a8e99f85","Matches source index","https://otto.de/robots.txt","https://www.otto.de/robots.txt","llms/otto.de.txt","921ef677137e8d078477a183038c2fd7135f6bfaec24487f80794363159543cc","921ef677137e8d078477a183038c2fd7135f6bfaec24487f80794363159543cc","Matches source index","https://otto.de/llms.txt","https://www.otto.de/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/otto.de.txt","llms/otto.de.txt","86abc355699826c5ac3f6b8c847f9889b7fa68fcc1176bad10a86892a8e99f85","921ef677137e8d078477a183038c2fd7135f6bfaec24487f80794363159543cc","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"167","bol.com","Bol","E-commerce & retail","Marketplace","Netherlands","Global","2026-10-21","https://www.bol.com/robots.txt","200","ok","6805","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bol.com.txt","1ba0b91c6d2d291e4b56a3350445c1b4a3e0467a526e902f6836611ef50fae80","1ba0b91c6d2d291e4b56a3350445c1b4a3e0467a526e902f6836611ef50fae80","Matches source index","https://bol.com/robots.txt","https://www.bol.com/robots.txt","","","84733b7c97f851b337c6467937426004e0758526496dc2e1dfd391238f2cd616","No archived body","https://bol.com/llms.txt","https://www.bol.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bol.com.txt","","1ba0b91c6d2d291e4b56a3350445c1b4a3e0467a526e902f6836611ef50fae80","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"168","cdiscount.com","Cdiscount","E-commerce & retail","Marketplace","France","Global","2026-10-21","https://www.cdiscount.com/robots.txt","200","ok","8269","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cdiscount.com.txt","a94b064ba37c5de2d014b73e00e329c39d9ddb8f8e52149cfe568aecbb8dccc7","a94b064ba37c5de2d014b73e00e329c39d9ddb8f8e52149cfe568aecbb8dccc7","Matches source index","https://cdiscount.com/robots.txt","https://www.cdiscount.com/robots.txt","","","79728a15bc392af867f5a35e760e781a81a7f707cd697beb84480beb114a8c99","No archived body","https://cdiscount.com/llms.txt","https://www.cdiscount.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cdiscount.com.txt","","a94b064ba37c5de2d014b73e00e329c39d9ddb8f8e52149cfe568aecbb8dccc7","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"169","allegro.pl","Allegro","E-commerce & retail","Marketplace","Poland","Global","2026-10-21","https://allegro.pl/robots.txt","200","ok","1363","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","55","has_strict_links","90","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/allegro.pl.txt","a1cd0a72831490645d1781c9ca5862091440544864843f5ff6c014313d6dd299","a1cd0a72831490645d1781c9ca5862091440544864843f5ff6c014313d6dd299","Matches source index","https://allegro.pl/robots.txt","https://allegro.pl/robots.txt","llms/allegro.pl.txt","d6bdece213e19f74ea74a310b77f35eab8c463a7da6128ae737b7173468c1a93","d6bdece213e19f74ea74a310b77f35eab8c463a7da6128ae737b7173468c1a93","Matches source index","https://allegro.pl/llms.txt","https://allegro.pl/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/allegro.pl.txt","llms/allegro.pl.txt","a1cd0a72831490645d1781c9ca5862091440544864843f5ff6c014313d6dd299","d6bdece213e19f74ea74a310b77f35eab8c463a7da6128ae737b7173468c1a93","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"170","pinduoduo.com","Pinduoduo","E-commerce & retail","Marketplace","China","Global","2026-10-21","https://pinduoduo.com/robots.txt","200","ok","41","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pinduoduo.com.txt","a6cf7c026a64c8eb0915d0d1ebc2c9504f5ed902aec140f2dcaef1d779145742","a6cf7c026a64c8eb0915d0d1ebc2c9504f5ed902aec140f2dcaef1d779145742","Matches source index","https://pinduoduo.com/robots.txt","https://pinduoduo.com/robots.txt","llms/pinduoduo.com.txt","050340f5d7a29a2c17b7af7f512e866811511b18253dcb8de3b6c5bb00e1f5bb","050340f5d7a29a2c17b7af7f512e866811511b18253dcb8de3b6c5bb00e1f5bb","Matches source index","https://pinduoduo.com/llms.txt","https://pinduoduo.com/","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/pinduoduo.com.txt","llms/pinduoduo.com.txt","a6cf7c026a64c8eb0915d0d1ebc2c9504f5ed902aec140f2dcaef1d779145742","050340f5d7a29a2c17b7af7f512e866811511b18253dcb8de3b6c5bb00e1f5bb","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"171","jpmorganchase.com","JPMorgan Chase","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.jpmorganchase.com/robots.txt","200","ok","437","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jpmorganchase.com.txt","dd14ce56e033ae6c67ba3e6c7b778ea3fabc540c761bacdc588b6290581f9ebf","dd14ce56e033ae6c67ba3e6c7b778ea3fabc540c761bacdc588b6290581f9ebf","Matches source index","https://jpmorganchase.com/robots.txt","https://www.jpmorganchase.com/robots.txt","","","637a66840cb7a1be5880ab80389590b07dad6784f99819955fd877726ee89e8a","No archived body","https://jpmorganchase.com/llms.txt","https://www.jpmorganchase.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jpmorganchase.com.txt","","dd14ce56e033ae6c67ba3e6c7b778ea3fabc540c761bacdc588b6290581f9ebf","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"172","bankofamerica.com","Bank of America","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.bankofamerica.com/robots.txt","200","ok","3312","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bankofamerica.com.txt","a5e538b28463bf76e653bcdf6af94ae510991a3c4341cdcf7535de75d9f5a691","a5e538b28463bf76e653bcdf6af94ae510991a3c4341cdcf7535de75d9f5a691","Matches source index","https://bankofamerica.com/robots.txt","https://www.bankofamerica.com/robots.txt","","","18b000c6bf53c7f5eb55d3f9644756be725bec88a9cff187a249b84e5527a269","No archived body","https://bankofamerica.com/llms.txt","https://www.bankofamerica.com/banking-information/error-page-en.html","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bankofamerica.com.txt","","a5e538b28463bf76e653bcdf6af94ae510991a3c4341cdcf7535de75d9f5a691","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"173","wellsfargo.com","Wells Fargo","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.wellsfargo.com/robots.txt","200","ok","621","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wellsfargo.com.txt","bb328d936c5a9a4b60781cc252850514d8c577405635cc6b9207bb59515d251d","bb328d936c5a9a4b60781cc252850514d8c577405635cc6b9207bb59515d251d","Matches source index","https://wellsfargo.com/robots.txt","https://www.wellsfargo.com/robots.txt","","","87d99d5848f39cf5814e7ad62e1c0c42423b1a36e39e0718c51e7d66abe845e2","No archived body","https://wellsfargo.com/llms.txt","https://www.wellsfargo.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/wellsfargo.com.txt","","bb328d936c5a9a4b60781cc252850514d8c577405635cc6b9207bb59515d251d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"174","citi.com","Citi","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.citi.com/robots.txt","200","ok","955","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/citi.com.txt","5100d200ce872b7516043de65a2fde69d94036a35ca72650fa9c9e474497909e","5100d200ce872b7516043de65a2fde69d94036a35ca72650fa9c9e474497909e","Matches source index","https://citi.com/robots.txt","https://www.citi.com/robots.txt","","","ff19b29ab26616e49fee0bdba5d5e8dc708724681793d40c38e8381072a6a358","No archived body","https://www.citi.com/llms.txt","https://www.citi.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/citi.com.txt","","5100d200ce872b7516043de65a2fde69d94036a35ca72650fa9c9e474497909e","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"175","goldmansachs.com","Goldman Sachs","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.goldmansachs.com/robots.txt","200","ok","635","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/goldmansachs.com.txt","05291ff5cd6f0670c69a5a45e0c62135221dedc77844aac77874e6c38f571523","05291ff5cd6f0670c69a5a45e0c62135221dedc77844aac77874e6c38f571523","Matches source index","https://goldmansachs.com/robots.txt","https://www.goldmansachs.com/robots.txt","","","8a95694c2b868629a27b1d2e5f8a5be90df0f16b2fa49dad61afa057cad4848d","No archived body","https://goldmansachs.com/llms.txt","https://www.goldmansachs.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/goldmansachs.com.txt","","05291ff5cd6f0670c69a5a45e0c62135221dedc77844aac77874e6c38f571523","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"176","morganstanley.com","Morgan Stanley","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.morganstanley.com/robots.txt","200","ok","1811","valid_txt_200","True","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/morganstanley.com.txt","9e029021331fa857ddb0f59d97f6b45cdcf58f287dc7e724ddcd1e33fac1c25d","9e029021331fa857ddb0f59d97f6b45cdcf58f287dc7e724ddcd1e33fac1c25d","Matches source index","https://morganstanley.com/robots.txt","https://www.morganstanley.com/robots.txt","","","d54c35646901a91cdb524da97044330074146d6751412526f7062a7162e82353","No archived body","https://morganstanley.com/llms.txt","https://www.morganstanley.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/morganstanley.com.txt","","9e029021331fa857ddb0f59d97f6b45cdcf58f287dc7e724ddcd1e33fac1c25d","","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"177","americanexpress.com","American Express","Finance, banking & insurance","Card network","United States","Global","2026-10-22","https://www.americanexpress.com/robots.txt","200","ok","298","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","1","has_strict_links","2","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/americanexpress.com.txt","9af422f9f806065d60668774fde9ac3806e61e7acf3043c84b0cd85aede4ea8f","9af422f9f806065d60668774fde9ac3806e61e7acf3043c84b0cd85aede4ea8f","Matches source index","https://americanexpress.com/robots.txt","https://www.americanexpress.com/robots.txt","llms/americanexpress.com.txt","5ef7e1544b2861c5b581d24bd71693abaafede327ca122c917a452d1f10e1026","5ef7e1544b2861c5b581d24bd71693abaafede327ca122c917a452d1f10e1026","Matches source index","https://americanexpress.com/llms.txt","https://www.americanexpress.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/americanexpress.com.txt","llms/americanexpress.com.txt","9af422f9f806065d60668774fde9ac3806e61e7acf3043c84b0cd85aede4ea8f","5ef7e1544b2861c5b581d24bd71693abaafede327ca122c917a452d1f10e1026","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"178","visa.com","Visa","Finance, banking & insurance","Card network","United States","Global","2026-10-21","https://www.visa.com/robots.txt","200","ok","0","empty_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/visa.com.txt","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","Matches source index","https://visa.com/robots.txt","https://www.visa.com/robots.txt","","","ed5305e1b589e1478173fa6038ce5cc90328349c2e7f99e73e1a5069238bf869","No archived body","https://visa.com/llms.txt","https://www.visa.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/visa.com.txt","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"179","mastercard.com","Mastercard","Finance, banking & insurance","Card network","United States","Global","2026-10-22","https://mastercard.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://mastercard.com/robots.txt","https://mastercard.com/robots.txt","","","1990427740973e4bbd58d9832d0925beb836ce62229243cc7018714430e1df5a","No archived body","https://mastercard.com/llms.txt","https://www.mastercard.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-99e2c34b3ef5","2026-10-22 23:49:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-bca4566a48a2","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"180","paypal.com","PayPal","Finance, banking & insurance","Fintech","United States","Global","2026-10-21","https://www.paypal.com/robots.txt","200","ok","2628","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","152","has_strict_links","152","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/paypal.com.txt","9c2f2f57e870e9b8406b1b738f0afbae6434f9991f24812890de2a6e317fc083","9c2f2f57e870e9b8406b1b738f0afbae6434f9991f24812890de2a6e317fc083","Matches source index","https://paypal.com/robots.txt","https://www.paypal.com/robots.txt","llms/paypal.com.txt","1e1be425396507ad6c9bdecdd342c9ada72019ec9a0572ec40c5804122932fa8","1e1be425396507ad6c9bdecdd342c9ada72019ec9a0572ec40c5804122932fa8","Matches source index","https://paypal.com/llms.txt","https://www.paypal.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/paypal.com.txt","llms/paypal.com.txt","9c2f2f57e870e9b8406b1b738f0afbae6434f9991f24812890de2a6e317fc083","1e1be425396507ad6c9bdecdd342c9ada72019ec9a0572ec40c5804122932fa8","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"181","capitalone.com","Capital One","Finance, banking & insurance","Bank","United States","Global","2026-10-21","https://www.capitalone.com/robots.txt","200","ok","2664","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/capitalone.com.txt","ca918affc355729be575001190da784c35c36eab16d61d4f2b7c0ec91118ecd1","ca918affc355729be575001190da784c35c36eab16d61d4f2b7c0ec91118ecd1","Matches source index","https://capitalone.com/robots.txt","https://www.capitalone.com/robots.txt","","","e641dea7daf4cbd3ee5eda91d5ca57c6961e686da961a93f0267fb23931d6b67","No archived body","https://capitalone.com/llms.txt","https://www.capitalone.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/capitalone.com.txt","","ca918affc355729be575001190da784c35c36eab16d61d4f2b7c0ec91118ecd1","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"182","discover.com","Discover","Finance, banking & insurance","Card network","United States","Global","2026-10-21","https://www.discover.com/robots.txt","200","ok","2107","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/discover.com.txt","bb97c7f0adf87b96838e83bd596d52d51f3710f583c2df0e8e1b8130b4836e7c","bb97c7f0adf87b96838e83bd596d52d51f3710f583c2df0e8e1b8130b4836e7c","Matches source index","https://discover.com/robots.txt","https://www.discover.com/robots.txt","","","b2d58088f9e7fc15c8263690b99b9e935b161148504c7eba3c83d3d3e8fe8380","No archived body","https://discover.com/llms.txt","https://www.discover.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/discover.com.txt","","bb97c7f0adf87b96838e83bd596d52d51f3710f583c2df0e8e1b8130b4836e7c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"183","hsbc.com","HSBC","Finance, banking & insurance","Bank","United Kingdom","Global","2026-10-21","https://www.hsbc.com/robots.txt","200","ok","24","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hsbc.com.txt","2c0f76ae2d26f13dbdfc883e6814ac8b7d67f808fd3ab15f182b33327f72427f","2c0f76ae2d26f13dbdfc883e6814ac8b7d67f808fd3ab15f182b33327f72427f","Matches source index","https://hsbc.com/robots.txt","https://www.hsbc.com/robots.txt","","","85a01fcf963612b617e0bf340a8470317f4934ec5259470a26495cb111216568","No archived body","https://hsbc.com/llms.txt","https://www.hsbc.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hsbc.com.txt","","2c0f76ae2d26f13dbdfc883e6814ac8b7d67f808fd3ab15f182b33327f72427f","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"184","barclays.com","Barclays","Finance, banking & insurance","Bank","United Kingdom","Global","2026-10-21","https://home.barclays/robots.txt","200","ok","83","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host home.barclays","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/barclays.com.txt","9f3d2cd8166a2fe6d63ecb7d1ef914804be871c6ded7b1b1308b8f8f52625eea","9f3d2cd8166a2fe6d63ecb7d1ef914804be871c6ded7b1b1308b8f8f52625eea","Matches source index","https://barclays.com/robots.txt","https://home.barclays/robots.txt","","","0beb61770e25889783d9e3c6f180e560f4c833680bbde0b4594e78d3eb8f5296","No archived body","https://barclays.com/llms.txt","https://home.barclays/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/barclays.com.txt","","9f3d2cd8166a2fe6d63ecb7d1ef914804be871c6ded7b1b1308b8f8f52625eea","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"185","lloydsbank.com","Lloyds Bank","Finance, banking & insurance","Bank","United Kingdom","Global","2026-10-22","https://www.lloydsbank.com/robots.txt","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/lloydsbank.com.txt","9391ae6632df8d61189d5cccba8b1999730c4d8da532909e53160999cc7c9baf","","Computed from archived invalid body; source index hash absent","https://lloydsbank.com/robots.txt","https://www.lloydsbank.com/robots.txt","llms/lloydsbank.com.txt","3add7b17dee1f87d961d83efa7ee20c2918f26388bacdced6e456f20293ca7ab","3add7b17dee1f87d961d83efa7ee20c2918f26388bacdced6e456f20293ca7ab","Matches source index","https://lloydsbank.com/llms.txt","https://www.lloydsbank.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/lloydsbank.com.txt","llms/lloydsbank.com.txt","9391ae6632df8d61189d5cccba8b1999730c4d8da532909e53160999cc7c9baf","3add7b17dee1f87d961d83efa7ee20c2918f26388bacdced6e456f20293ca7ab","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222349-2533-0275-0000-e42bd0b36d30","2026-10-22 23:49:45 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","","","","","","",""
"186","santander.com","Santander","Finance, banking & insurance","Bank","Spain","Global","2026-10-21","https://www.santander.com/robots.txt","200","ok","729","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/santander.com.txt","7a817b79eb7308b844f3422336aab43c96e979fcc880044a54917fda708d0c72","7a817b79eb7308b844f3422336aab43c96e979fcc880044a54917fda708d0c72","Matches source index","https://santander.com/robots.txt","https://www.santander.com/robots.txt","","","b5f272c41a300a550f112f5d3938357f99a41e0e2e86240fb0a7c2ac3131c1ef","No archived body","https://santander.com/llms.txt","https://www.santander.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/santander.com.txt","","7a817b79eb7308b844f3422336aab43c96e979fcc880044a54917fda708d0c72","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"187","bnpparibas.com","BNP Paribas","Finance, banking & insurance","Bank","France","Global","2026-10-22","https://bnpparibas.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://bnpparibas.com/robots.txt","https://bnpparibas.com/robots.txt","","","2b9737d1e07d940cc1244dc0e8de27277ee6abbce3a3a2cc08007c5f00acb49f","No archived body","https://bnpparibas.com/llms.txt","https://bnpparibas.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-616398fd9675","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-ceec5f9c86f0","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"188","ing.com","ING","Finance, banking & insurance","Bank","Netherlands","Global","2026-10-21","https://ing.com/robots.txt","200","ok","127","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ing.com.txt","e994dad16dd3b419e90c4acc9e112be2ce89ebe062411137fbbb6567f31ad821","e994dad16dd3b419e90c4acc9e112be2ce89ebe062411137fbbb6567f31ad821","Matches source index","https://ing.com/robots.txt","https://ing.com/robots.txt","","","2170cdf1f569aadf0b95637dadd7b365ee7f16a1b53959be5f0eac79a31fe460","No archived body","https://ing.com/llms.txt","https://ing.com/llms.txt","404","text/html;charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ing.com.txt","","e994dad16dd3b419e90c4acc9e112be2ce89ebe062411137fbbb6567f31ad821","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"189","ubs.com","UBS","Finance, banking & insurance","Bank","Switzerland","Global","2026-10-22","https://www.ubs.com/robots.txt","200","ok","1293","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","360","has_strict_links","360","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/ubs.com.txt","ce5082f5aa6d2de2867765d840eb1709a5b4cfd2bc462ee54fae8f16ecafdddc","ce5082f5aa6d2de2867765d840eb1709a5b4cfd2bc462ee54fae8f16ecafdddc","Matches source index","https://ubs.com/robots.txt","https://www.ubs.com/robots.txt","llms/ubs.com.txt","0f10e3a94318f54548d13ab7bffbe584e376fc06310d26e1807b1e8c13aae7ae","0f10e3a94318f54548d13ab7bffbe584e376fc06310d26e1807b1e8c13aae7ae","Matches source index","https://ubs.com/llms.txt","https://www.ubs.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","present","Detected (strict MIME)","raw/ubs.com.txt","llms/ubs.com.txt","ce5082f5aa6d2de2867765d840eb1709a5b4cfd2bc462ee54fae8f16ecafdddc","0f10e3a94318f54548d13ab7bffbe584e376fc06310d26e1807b1e8c13aae7ae","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"190","deutschebank.com","Deutsche Bank","Finance, banking & insurance","Bank","Germany","Global","2026-10-22","https://deutschebank.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, Non ', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://deutschebank.com/robots.txt","https://deutschebank.com/robots.txt","","","","No archived body","https://deutschebank.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222349-2533-0275-0000-5593cac8852d","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","","10222351-2533-0275-0000-3ea08441199b","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"191","societegenerale.com","Societe Generale","Finance, banking & insurance","Bank","France","Global","2026-10-22","https://www.societegenerale.com/robots.txt","200","ok","1654","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/societegenerale.com.txt","399e4ae3a8fc957cb61f086467cedea223a5ca66c2656a9e31b25b60a4375521","399e4ae3a8fc957cb61f086467cedea223a5ca66c2656a9e31b25b60a4375521","Matches source index","https://societegenerale.com/robots.txt","https://www.societegenerale.com/robots.txt","","","f12e983aeea7e4f51ed819501f841920a72b7a8c81aa56db6956a2fec98f7ab2","No archived body","https://societegenerale.com/llms.txt","https://www.societegenerale.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/societegenerale.com.txt","","399e4ae3a8fc957cb61f086467cedea223a5ca66c2656a9e31b25b60a4375521","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"192","bbva.com","BBVA","Finance, banking & insurance","Bank","Spain","Global","2026-10-21","https://www.bbva.com/robots.txt","200","ok","911","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bbva.com.txt","c158dc0b182580b5135f12b987dcee1784a6fb724a3280d0ffe03e03310f2489","c158dc0b182580b5135f12b987dcee1784a6fb724a3280d0ffe03e03310f2489","Matches source index","https://bbva.com/robots.txt","https://www.bbva.com/robots.txt","","","63f0885c46dc849cc2c9644f0226009c7a1e29636e012cefd61eb7ac042f240a","No archived body","https://bbva.com/llms.txt","https://www.bbva.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bbva.com.txt","","c158dc0b182580b5135f12b987dcee1784a6fb724a3280d0ffe03e03310f2489","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"193","unicreditgroup.eu","UniCredit","Finance, banking & insurance","Bank","Italy","Global","2026-10-21","https://www.unicreditgroup.eu/robots.txt","200","ok","2206","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","11","has_strict_links","12","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/unicreditgroup.eu.txt","fe556440aff69c03b8a2184eee2cabac2399a0fc8968951aaa60fef62a6873f1","fe556440aff69c03b8a2184eee2cabac2399a0fc8968951aaa60fef62a6873f1","Matches source index","https://unicreditgroup.eu/robots.txt","https://www.unicreditgroup.eu/robots.txt","llms/unicreditgroup.eu.txt","97e4348397affb77669a47980c296ba4ef67ed08b74122685e88bf105d505b3a","97e4348397affb77669a47980c296ba4ef67ed08b74122685e88bf105d505b3a","Matches source index","https://unicreditgroup.eu/llms.txt","https://www.unicreditgroup.eu/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/unicreditgroup.eu.txt","llms/unicreditgroup.eu.txt","fe556440aff69c03b8a2184eee2cabac2399a0fc8968951aaa60fef62a6873f1","97e4348397affb77669a47980c296ba4ef67ed08b74122685e88bf105d505b3a","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"194","allianz.com","Allianz","Finance, banking & insurance","Insurance","Germany","Global","2026-10-21","https://www.allianz.com:443/robots.txt","200","ok","577","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: robots.txt redirects to host www.allianz.com:443 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/allianz.com.txt","c7515fc5697f58b72b11757fe91d4824cdc28a4d27ebb1eb00691338b31f586d","c7515fc5697f58b72b11757fe91d4824cdc28a4d27ebb1eb00691338b31f586d","Matches source index","https://allianz.com/robots.txt","https://www.allianz.com:443/robots.txt","","","","No stored body; provider HTTP response in task record","https://allianz.com/llms.txt","https://www.allianz.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/allianz.com.txt","","c7515fc5697f58b72b11757fe91d4824cdc28a4d27ebb1eb00691338b31f586d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-95199be89ee7","2026-10-22 23:42:58 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"195","axa.com","AXA","Finance, banking & insurance","Insurance","France","Global","2026-10-21","https://www.axa.com/robots.txt","200","ok","236","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/axa.com.txt","4e20fa32384a572078e626777c732ad7fc6d1a249a15f9e52561a6b62007b5e5","4e20fa32384a572078e626777c732ad7fc6d1a249a15f9e52561a6b62007b5e5","Matches source index","https://axa.com/robots.txt","https://www.axa.com/robots.txt","","","0986b282bdd6892ac96f4c7de76c6a060d7cbfd02385698c92d7f5f5556ceb61","No archived body","https://axa.com/llms.txt","https://www.axa.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/axa.com.txt","","4e20fa32384a572078e626777c732ad7fc6d1a249a15f9e52561a6b62007b5e5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"196","metlife.com","MetLife","Finance, banking & insurance","Insurance","United States","Global","2026-10-21","https://www.metlife.com/robots.txt","200","ok","2036","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","8","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/metlife.com.txt","b5f6347b74166fc03b1a646ae74b38666aa522f2d6d31ba42dfbcf18f47832ca","b5f6347b74166fc03b1a646ae74b38666aa522f2d6d31ba42dfbcf18f47832ca","Matches source index","https://metlife.com/robots.txt","https://www.metlife.com/robots.txt","llms/metlife.com.txt","00d693f727268d2e114094107dc565fa0a35bbdc5c22fb1085560bc22131ce36","00d693f727268d2e114094107dc565fa0a35bbdc5c22fb1085560bc22131ce36","Matches source index","https://metlife.com/llms.txt","https://www.metlife.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/metlife.com.txt","llms/metlife.com.txt","b5f6347b74166fc03b1a646ae74b38666aa522f2d6d31ba42dfbcf18f47832ca","00d693f727268d2e114094107dc565fa0a35bbdc5c22fb1085560bc22131ce36","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"197","prudential.com","Prudential Financial","Finance, banking & insurance","Insurance","United States","Global","2026-10-22","https://www.prudential.com/robots.txt","200","ok","5614","valid_txt_200","True","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Max retries exceeded with url: /com'","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/prudential.com.txt","f7900048a2344c81939df9e6cb84e838904eba99abf098835c649b052a97ba77","f7900048a2344c81939df9e6cb84e838904eba99abf098835c649b052a97ba77","Matches source index","https://prudential.com/robots.txt","https://www.prudential.com/robots.txt","","","16e630b172321e26413af3a2bc74591c630f976044c0cc6cc2c9d8138495295c","No archived body","https://www.prudential.com/llms.txt","https://www.prudential.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/prudential.com.txt","","f7900048a2344c81939df9e6cb84e838904eba99abf098835c649b052a97ba77","","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"198","aig.com","AIG","Finance, banking & insurance","Insurance","United States","Global","2026-10-21","https://www.aig.com:443/robots.txt","200","ok","2212","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host www.aig.com:443","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aig.com.txt","a3dd85f8ba4ba9b3b4504ee9b0de4ba4648fb1ae77efdc3f2bd49b62bcacfd41","a3dd85f8ba4ba9b3b4504ee9b0de4ba4648fb1ae77efdc3f2bd49b62bcacfd41","Matches source index","https://aig.com/robots.txt","https://www.aig.com:443/robots.txt","","","329a7fd283352769fdd46dfe9b23deefbd9437add1c891ab88fab16424d9ea00","No archived body","https://aig.com/llms.txt","https://www.aig.com:443/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aig.com.txt","","a3dd85f8ba4ba9b3b4504ee9b0de4ba4648fb1ae77efdc3f2bd49b62bcacfd41","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"199","progressive.com","Progressive","Finance, banking & insurance","Insurance","United States","Global","2026-10-21","https://www.progressive.com/robots.txt","200","ok","1919","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/progressive.com.txt","c5f96db2be2961734460c2684457eb09e5226afc8fdca868b5a60a5a460eb1bb","c5f96db2be2961734460c2684457eb09e5226afc8fdca868b5a60a5a460eb1bb","Matches source index","https://progressive.com/robots.txt","https://www.progressive.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://www.progressive.com/llms.txt","https://www.progressive.com/llms.txt","404","text/HTML","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/progressive.com.txt","","c5f96db2be2961734460c2684457eb09e5226afc8fdca868b5a60a5a460eb1bb","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222351-2533-0275-0000-7f80570db236","2026-10-22 23:51:56 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"200","geico.com","GEICO","Finance, banking & insurance","Insurance","United States","Global","2026-10-21","https://www.geico.com/robots.txt","200","ok","1526","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","20","has_strict_links","20","True","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/geico.com.txt","7f45fe6ad5aee87aae29c1ce02aaf8ab807d1c136981ada7de9edf027eee5b01","7f45fe6ad5aee87aae29c1ce02aaf8ab807d1c136981ada7de9edf027eee5b01","Matches source index","https://geico.com/robots.txt","https://www.geico.com/robots.txt","crawler-api-verification-pass/raw/llms/geico.com--10222342-2533-0275-0000-fc9f7e37d6a7.txt","544f784e23c9a0c02786cb12702275d104def8ed8d3efcd55720e757dc1a06e6","","API-returned UTF-8 body; byte length matched provider size","https://geico.com/llms.txt","https://www.geico.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/geico.com.txt","","7f45fe6ad5aee87aae29c1ce02aaf8ab807d1c136981ada7de9edf027eee5b01","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222342-2533-0275-0000-fc9f7e37d6a7","2026-10-22 23:42:59 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-03.json","1"
"201","statefarm.com","State Farm","Finance, banking & insurance","Insurance","United States","Global","2026-10-22","https://www.statefarm.com/robots.txt","200","ok","1005","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/statefarm.com.txt","a5e2522c0557efbda7ea4cf73f72f044a3a9d4b67f10c1d87ce8d68366301c32","a5e2522c0557efbda7ea4cf73f72f044a3a9d4b67f10c1d87ce8d68366301c32","Matches source index","https://statefarm.com/robots.txt","https://www.statefarm.com/robots.txt","","","d110623d44f6f3447c44f5ee5865099a6317d6983f013bbef6219961d87fcdc9","No archived body","https://www.statefarm.com/llms.txt","https://www.statefarm.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/statefarm.com.txt","","a5e2522c0557efbda7ea4cf73f72f044a3a9d4b67f10c1d87ce8d68366301c32","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"202","fidelity.com","Fidelity Investments","Finance, banking & insurance","Investment","United States","Global","2026-10-21","https://www.fidelity.com:443/robots.txt","200","ok","12403","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note: robots.txt redirects to host www.fidelity.com:443 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/fidelity.com.txt","b0d6dc0a3b99913ed2e51e3460998e0b49fe141b1e7bbce202f1aabea1c51eb9","b0d6dc0a3b99913ed2e51e3460998e0b49fe141b1e7bbce202f1aabea1c51eb9","Matches source index","https://fidelity.com/robots.txt","https://www.fidelity.com:443/robots.txt","","","0397ae5cdc2975ddcdb54568cb3fb7fd6e5921e1af968227b6ab76a30238ab5d","No archived body","https://fidelity.com/llms.txt","https://www.fidelity.com:443/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/fidelity.com.txt","","b0d6dc0a3b99913ed2e51e3460998e0b49fe141b1e7bbce202f1aabea1c51eb9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-43ab266038d5","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"203","vanguard.com","Vanguard","Finance, banking & insurance","Investment","United States","Global","2026-10-21","https://www.vanguard.com/robots.txt","200","ok","940","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/vanguard.com.txt","c47befa7c174a485a302850f0b4c8923c657fba602c2586c499b247a14d34c67","c47befa7c174a485a302850f0b4c8923c657fba602c2586c499b247a14d34c67","Matches source index","https://vanguard.com/robots.txt","https://www.vanguard.com/robots.txt","","","7f005bc9f12d93ae33a26760f6d1e94b12c9bb2f648126d01f710db661b5463f","No archived body","https://vanguard.com/llms.txt","https://www.vanguard.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/vanguard.com.txt","","c47befa7c174a485a302850f0b4c8923c657fba602c2586c499b247a14d34c67","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"204","schwab.com","Charles Schwab","Finance, banking & insurance","Investment","United States","Global","2026-10-21","https://www.schwab.com/robots.txt","200","ok","1314","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/schwab.com.txt","4a83bd72dd08f2733580d946b05df8b5415d1dff4465c77592f6053df117c945","4a83bd72dd08f2733580d946b05df8b5415d1dff4465c77592f6053df117c945","Matches source index","https://schwab.com/robots.txt","https://www.schwab.com/robots.txt","","","3220527aa388514597a008d74eb1aa8f231bc4f149e5042e41fd999ca634f265","No archived body","https://schwab.com/llms.txt","https://www.schwab.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/schwab.com.txt","","4a83bd72dd08f2733580d946b05df8b5415d1dff4465c77592f6053df117c945","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"205","blackrock.com","BlackRock","Finance, banking & insurance","Investment","United States","Global","2026-10-21","https://www.blackrock.com/robots.txt","200","ok","5506","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/blackrock.com.txt","17c5a9d36cc5bfec5d7162e2f047bdfed7e294cd9e19246699cb0a40bd224868","17c5a9d36cc5bfec5d7162e2f047bdfed7e294cd9e19246699cb0a40bd224868","Matches source index","https://blackrock.com/robots.txt","https://www.blackrock.com/robots.txt","","","70bda9359821716ee8cbc83f68339bce544ed4b39fdfbb95ca635439d3f5fca1","No archived body","https://blackrock.com/llms.txt","https://www.blackrock.com/corporate/en-us/404-not-found","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/blackrock.com.txt","","17c5a9d36cc5bfec5d7162e2f047bdfed7e294cd9e19246699cb0a40bd224868","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"206","stripe.com","Stripe","Finance, banking & insurance","Fintech","","Global","2026-10-21","https://stripe.com/robots.txt","200","ok","643","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","308","has_strict_links","308","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/stripe.com.txt","23166bdd2477d69a2fbd405ae5901154a36f5de79876de4616494458fb340002","23166bdd2477d69a2fbd405ae5901154a36f5de79876de4616494458fb340002","Matches source index","https://stripe.com/robots.txt","https://stripe.com/robots.txt","llms/stripe.com.txt","d1251f7b78e9f0d58ccb13f6dfe409d0950c9937899031f22e405c6865d9d9b4","d1251f7b78e9f0d58ccb13f6dfe409d0950c9937899031f22e405c6865d9d9b4","Matches source index","https://stripe.com/llms.txt","https://stripe.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/stripe.com.txt","llms/stripe.com.txt","23166bdd2477d69a2fbd405ae5901154a36f5de79876de4616494458fb340002","d1251f7b78e9f0d58ccb13f6dfe409d0950c9937899031f22e405c6865d9d9b4","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"207","squareup.com","Square","Finance, banking & insurance","Fintech","United States","Global","2026-10-21","https://squareup.com/robots.txt","200","ok","2199","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","169","has_strict_links","179","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/squareup.com.txt","7fa638b4028a50d503304cf33461506de7b6b67f262b59eb6d737be6240a75f4","7fa638b4028a50d503304cf33461506de7b6b67f262b59eb6d737be6240a75f4","Matches source index","https://squareup.com/robots.txt","https://squareup.com/robots.txt","llms/squareup.com.txt","7b3033b2522a8bf95abf6e289d41756a02760cc066e04a827649ec869768ed0d","7b3033b2522a8bf95abf6e289d41756a02760cc066e04a827649ec869768ed0d","Matches source index","https://squareup.com/llms.txt","https://squareup.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/squareup.com.txt","llms/squareup.com.txt","7fa638b4028a50d503304cf33461506de7b6b67f262b59eb6d737be6240a75f4","7b3033b2522a8bf95abf6e289d41756a02760cc066e04a827649ec869768ed0d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"208","robinhood.com","Robinhood","Finance, banking & insurance","Fintech","United States","Global","2026-10-21","https://robinhood.com/robots.txt","200","ok","240","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/robinhood.com.txt","85d4a1e5480598d2e8b31c48be15f6b79e05356416654e168cb9b6006e79ef4e","85d4a1e5480598d2e8b31c48be15f6b79e05356416654e168cb9b6006e79ef4e","Matches source index","https://robinhood.com/robots.txt","https://robinhood.com/robots.txt","llms/robinhood.com.txt","6f002849cf15dc073b8c5b46237bd8e9bd120f2f079bd746a64e71e9840fbb8a","6f002849cf15dc073b8c5b46237bd8e9bd120f2f079bd746a64e71e9840fbb8a","Matches source index","https://robinhood.com/llms.txt","https://robinhood.com/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/robinhood.com.txt","llms/robinhood.com.txt","85d4a1e5480598d2e8b31c48be15f6b79e05356416654e168cb9b6006e79ef4e","6f002849cf15dc073b8c5b46237bd8e9bd120f2f079bd746a64e71e9840fbb8a","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"209","coinbase.com","Coinbase","Finance, banking & insurance","Fintech","United States","Global","2026-10-21","https://www.coinbase.com/robots.txt","200","ok","3282","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","38","has_strict_links","41","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/coinbase.com.txt","7da87343a8d8b7782949de7da7d4b9162c2e26689606972e97228f576e6884f3","7da87343a8d8b7782949de7da7d4b9162c2e26689606972e97228f576e6884f3","Matches source index","https://coinbase.com/robots.txt","https://www.coinbase.com/robots.txt","llms/coinbase.com.txt","06ee994c899e49f79513ec3f30d2ccdfc5aa6ddd0f68c449336bee661a1ca6ab","06ee994c899e49f79513ec3f30d2ccdfc5aa6ddd0f68c449336bee661a1ca6ab","Matches source index","https://coinbase.com/llms.txt","https://www.coinbase.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/coinbase.com.txt","llms/coinbase.com.txt","7da87343a8d8b7782949de7da7d4b9162c2e26689606972e97228f576e6884f3","06ee994c899e49f79513ec3f30d2ccdfc5aa6ddd0f68c449336bee661a1ca6ab","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"210","rbc.com","Royal Bank of Canada","Finance, banking & insurance","Bank","Canada","Global","2026-10-21","https://www.rbc.com/robots.txt","200","ok","37","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/rbc.com.txt","b3d067885143e5ead08c085a53edeed739218e6cf67b26a2ed5142bfaa53c119","b3d067885143e5ead08c085a53edeed739218e6cf67b26a2ed5142bfaa53c119","Matches source index","https://rbc.com/robots.txt","https://www.rbc.com/robots.txt","","","5c6c66aa81297df4e3a4086273a15284c704a893c319bfa957bb3b77eabe3187","No archived body","https://rbc.com/llms.txt","https://www.rbc.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/rbc.com.txt","","b3d067885143e5ead08c085a53edeed739218e6cf67b26a2ed5142bfaa53c119","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"211","pfizer.com","Pfizer","Healthcare & pharma","Pharma","United States","Global","2026-10-21","https://www.pfizer.com/robots.txt","200","ok","1764","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pfizer.com.txt","bd8ad0bb0d547eec7466879755410c282296e64b3fe2547c85155c6a7aa54447","bd8ad0bb0d547eec7466879755410c282296e64b3fe2547c85155c6a7aa54447","Matches source index","https://pfizer.com/robots.txt","https://www.pfizer.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://pfizer.com/llms.txt","https://www.pfizer.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/pfizer.com.txt","","bd8ad0bb0d547eec7466879755410c282296e64b3fe2547c85155c6a7aa54447","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-4b7675a99340","2026-10-22 23:42:58 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"212","novartis.com","Novartis","Healthcare & pharma","Pharma","Switzerland","Global","2026-10-21","https://www.novartis.com/robots.txt","200","ok","1764","valid_txt_200","True","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/novartis.com.txt","c0e8d28ece534eeeef5553f2f36f5ae99c9315ad4afdc137ae02c21eca6f917c","c0e8d28ece534eeeef5553f2f36f5ae99c9315ad4afdc137ae02c21eca6f917c","Matches source index","https://novartis.com/robots.txt","https://www.novartis.com/robots.txt","","","9f59814cb08de6068fb8e1cf7c9890dc3a0bf45167d98a5ac8311a6c477b00ed","No archived body","https://novartis.com/llms.txt","https://www.novartis.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/novartis.com.txt","","c0e8d28ece534eeeef5553f2f36f5ae99c9315ad4afdc137ae02c21eca6f917c","","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"213","roche.com","Roche","Healthcare & pharma","Pharma","Switzerland","Global","2026-10-21","https://www.roche.com/robots.txt","200","ok","220","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/roche.com.txt","66f7c5f40e78962f0c883d7a914f83c8f6346e35b1af17a19fbdc0e27f8aea10","66f7c5f40e78962f0c883d7a914f83c8f6346e35b1af17a19fbdc0e27f8aea10","Matches source index","https://roche.com/robots.txt","https://www.roche.com/robots.txt","","","4b59e08136bdb2b367e5b4b3831c584fc8675f1126e20e5dfe0495dff039b4d9","No archived body","https://roche.com/llms.txt","https://www.roche.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/roche.com.txt","","66f7c5f40e78962f0c883d7a914f83c8f6346e35b1af17a19fbdc0e27f8aea10","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"214","merck.com","Merck & Co.","Healthcare & pharma","Pharma","United States","Global","2026-10-21","https://www.merck.com/robots.txt","200","ok","115","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/merck.com.txt","a50c0784730a141cdafb4867031b332f1f59508628ce9c0cfd8a5fe5a32ecedf","a50c0784730a141cdafb4867031b332f1f59508628ce9c0cfd8a5fe5a32ecedf","Matches source index","https://merck.com/robots.txt","https://www.merck.com/robots.txt","","","3b7dddc836e5bfd5b5b72c23b902b612ceee7751bbf5f0a436531b5f4bc1601a","No archived body","https://merck.com/llms.txt","https://www.merck.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/merck.com.txt","","a50c0784730a141cdafb4867031b332f1f59508628ce9c0cfd8a5fe5a32ecedf","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"215","astrazeneca.com","AstraZeneca","Healthcare & pharma","Pharma","United Kingdom","Global","2026-10-22","https://www.astrazeneca.com/robots.txt","200","ok","909","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","astrazeneca.com","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/astrazeneca.com.txt","cc7a0d6ef3372a63a77dcc85031459b0dd058a2f70ea48d9635e54417a11b456","cc7a0d6ef3372a63a77dcc85031459b0dd058a2f70ea48d9635e54417a11b456","Matches source index","https://astrazeneca.com/robots.txt","https://www.astrazeneca.com/robots.txt","llms/astrazeneca.com.txt","6d85b1ff8a1bb002950204277fec6cf4b4c0b913994eab4c5f33bbe3ccf0f6e8","6d85b1ff8a1bb002950204277fec6cf4b4c0b913994eab4c5f33bbe3ccf0f6e8","Matches source index","https://www.astrazeneca.com/llms.txt","https://www.astrazeneca.com/404.html","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/astrazeneca.com.txt","llms/astrazeneca.com.txt","cc7a0d6ef3372a63a77dcc85031459b0dd058a2f70ea48d9635e54417a11b456","6d85b1ff8a1bb002950204277fec6cf4b4c0b913994eab4c5f33bbe3ccf0f6e8","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"216","gsk.com","GSK","Healthcare & pharma","Pharma","United Kingdom","Global","2026-10-21","https://gsk.com/robots.txt","200","ok","81","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gsk.com.txt","37c4c025c95e16cd9d5f8c5ad0466fc58e3682829f0de96e02ff5007a6185447","37c4c025c95e16cd9d5f8c5ad0466fc58e3682829f0de96e02ff5007a6185447","Matches source index","https://gsk.com/robots.txt","https://gsk.com/robots.txt","llms/gsk.com.txt","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","Matches source index","https://gsk.com/llms.txt","https://www.gsk.com:443/llms.txt","200","","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gsk.com.txt","llms/gsk.com.txt","37c4c025c95e16cd9d5f8c5ad0466fc58e3682829f0de96e02ff5007a6185447","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"217","sanofi.com","Sanofi","Healthcare & pharma","Pharma","France","Global","2026-10-21","https://www.sanofi.com/robots.txt","200","ok","74","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/sanofi.com.txt","d25797ae63ba325f83a4082066c71c87129fc98982a9c83e32a908a4c9da9205","d25797ae63ba325f83a4082066c71c87129fc98982a9c83e32a908a4c9da9205","Matches source index","https://sanofi.com/robots.txt","https://www.sanofi.com/robots.txt","","","bbe2f23741be6a821ad36c33f3ddf008633ade583d870fe61332ddc3ca294532","No archived body","https://sanofi.com/llms.txt","https://www.sanofi.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/sanofi.com.txt","","d25797ae63ba325f83a4082066c71c87129fc98982a9c83e32a908a4c9da9205","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"218","bayer.com","Bayer","Healthcare & pharma","Pharma","Germany","Global","2026-10-22","https://bayer.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://bayer.com/robots.txt","https://bayer.com/robots.txt","","","7f0c4894292d51e585fbeefa93ab36e69b2edc33be002e1e8302ba9a2f59d97d","No archived body","https://bayer.com/llms.txt","https://www.bayer.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-2e4f22765077","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-10266f17d1e0","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"219","jnj.com","Johnson & Johnson","Healthcare & pharma","Pharma","United States","Global","2026-10-22","https://www.jnj.com/robots.txt","200","ok","2340","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jnj.com.txt","6418c413dc559473e07f9390980446e80292a89c3966ecd0757f2b864d8ab9a0","6418c413dc559473e07f9390980446e80292a89c3966ecd0757f2b864d8ab9a0","Matches source index","https://jnj.com/robots.txt","https://www.jnj.com/robots.txt","llms/jnj.com.txt","199b4c5ed44377f413cc20da3477db9ab7bd81031c7f81cb68b72b8d34aef128","199b4c5ed44377f413cc20da3477db9ab7bd81031c7f81cb68b72b8d34aef128","Matches source index","https://jnj.com/llms.txt","https://www.jnj.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/jnj.com.txt","llms/jnj.com.txt","6418c413dc559473e07f9390980446e80292a89c3966ecd0757f2b864d8ab9a0","199b4c5ed44377f413cc20da3477db9ab7bd81031c7f81cb68b72b8d34aef128","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"220","abbvie.com","AbbVie","Healthcare & pharma","Pharma","United States","Global","2026-10-22","https://www.abbvie.com/robots.txt","200","ok","1723","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/abbvie.com.txt","1361af4e0db66814f8e7db628a77f71798398ab513ffd13ff43b1bf69b8a5cdc","1361af4e0db66814f8e7db628a77f71798398ab513ffd13ff43b1bf69b8a5cdc","Matches source index","https://abbvie.com/robots.txt","https://www.abbvie.com/robots.txt","","","b52989760e17536f89c60405fd979a7faf42182b42277f2f8cbaa1b1e0ec56c9","No archived body","https://abbvie.com/llms.txt","https://www.abbvie.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/abbvie.com.txt","","1361af4e0db66814f8e7db628a77f71798398ab513ffd13ff43b1bf69b8a5cdc","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"221","amgen.com","Amgen","Healthcare & pharma","Biotech","United States","Global","2026-10-21","https://www.amgen.com/robots.txt","200","ok","117","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/amgen.com.txt","65f0f6f97ff9290dcb2dce7396adb326cd3a1ed30d798d5eca311d6f9bdca834","65f0f6f97ff9290dcb2dce7396adb326cd3a1ed30d798d5eca311d6f9bdca834","Matches source index","https://amgen.com/robots.txt","https://www.amgen.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://amgen.com/llms.txt","https://www.amgen.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/amgen.com.txt","","65f0f6f97ff9290dcb2dce7396adb326cd3a1ed30d798d5eca311d6f9bdca834","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"222","gilead.com","Gilead Sciences","Healthcare & pharma","Pharma","United States","Global","2026-10-21","https://www.gilead.com/robots.txt","200","ok","2688","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gilead.com.txt","bc1516c5bf59669a53b0dc4c12a8b2bd674f37656b032007a9813bb8afd11d5d","bc1516c5bf59669a53b0dc4c12a8b2bd674f37656b032007a9813bb8afd11d5d","Matches source index","https://gilead.com/robots.txt","https://www.gilead.com/robots.txt","","","0206dc325555c9b5859d3f5f2f0f97e1ee1406f0f2ea5a7f2bee08ad9377acc5","No archived body","https://gilead.com/llms.txt","https://www.gilead.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gilead.com.txt","","bc1516c5bf59669a53b0dc4c12a8b2bd674f37656b032007a9813bb8afd11d5d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"223","lilly.com","Eli Lilly","Healthcare & pharma","Pharma","United States","Global","2026-10-22","https://www.lilly.com/robots.txt","200","ok","1977","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/lilly.com.txt","6bbc9786f58ee56ca1f34a61ef31b34a11da920e91725e0f06b3d6616adf59f9","6bbc9786f58ee56ca1f34a61ef31b34a11da920e91725e0f06b3d6616adf59f9","Matches source index","https://lilly.com/robots.txt","https://www.lilly.com/robots.txt","","","3380f8d95fdea8b1facab1cc5dd34a92decb8142687204ae45b2438dbfcd3bec","No archived body","https://lilly.com/llms.txt","https://www.lilly.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/lilly.com.txt","","6bbc9786f58ee56ca1f34a61ef31b34a11da920e91725e0f06b3d6616adf59f9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"224","novonordisk.com","Novo Nordisk","Healthcare & pharma","Pharma","Denmark","Global","2026-10-21","https://www.novonordisk.com/robots.txt","200","ok","48","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/novonordisk.com.txt","370253490419b0a5c846c92c6959bf74d585866704d05fc79b76f364c0ae895b","370253490419b0a5c846c92c6959bf74d585866704d05fc79b76f364c0ae895b","Matches source index","https://novonordisk.com/robots.txt","https://www.novonordisk.com/robots.txt","","","ce3619efe6592f5af7415359f3a87232308d4ac914a509eea94a38ae3f55c9a2","No archived body","https://novonordisk.com/llms.txt","https://www.novonordisk.com/llms.txt","404","text/plain; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/novonordisk.com.txt","","370253490419b0a5c846c92c6959bf74d585866704d05fc79b76f364c0ae895b","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"225","takeda.com","Takeda","Healthcare & pharma","Pharma","Japan","Global","2026-10-21","https://www.takeda.com/robots.txt","200","ok","6361","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","0","no_strict_links","77","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/takeda.com.txt","a89c7e849a67c9d9e9432b02c53da5fbc725002891cac7153fbe96b895126698","a89c7e849a67c9d9e9432b02c53da5fbc725002891cac7153fbe96b895126698","Matches source index","https://takeda.com/robots.txt","https://www.takeda.com/robots.txt","llms/takeda.com.txt","8aef89fee9e7a5c441618477c418ad777730346cec31a78e57500e1d0c3e480c","8aef89fee9e7a5c441618477c418ad777730346cec31a78e57500e1d0c3e480c","Matches source index","https://takeda.com/llms.txt","https://www.takeda.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/takeda.com.txt","llms/takeda.com.txt","a89c7e849a67c9d9e9432b02c53da5fbc725002891cac7153fbe96b895126698","8aef89fee9e7a5c441618477c418ad777730346cec31a78e57500e1d0c3e480c","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"226","boehringer-ingelheim.com","Boehringer Ingelheim","Healthcare & pharma","Pharma","Germany","Global","2026-10-21","https://www.boehringer-ingelheim.com/robots.txt","200","ok","3055","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/boehringer-ingelheim.com.txt","2ad1ff0b9b121e91790939f67459d9192fd7d4e68f287ce7dddcea7a1f69dc85","2ad1ff0b9b121e91790939f67459d9192fd7d4e68f287ce7dddcea7a1f69dc85","Matches source index","https://boehringer-ingelheim.com/robots.txt","https://www.boehringer-ingelheim.com/robots.txt","","","3f8d0c44e7e502b93a879c1f78d1c9990a2d4c508c6ee614aa99ec0b89e60e44","No archived body","https://boehringer-ingelheim.com/llms.txt","https://www.boehringer-ingelheim.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/boehringer-ingelheim.com.txt","","2ad1ff0b9b121e91790939f67459d9192fd7d4e68f287ce7dddcea7a1f69dc85","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-a34520ef46c8","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"227","moderna.com","Moderna","Healthcare & pharma","Biotech","United States","Global","2026-10-21","https://moderna.com/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://moderna.com/robots.txt","https://moderna.com/robots.txt","","","b9347f234dc3c8d56e015e86d88a1400415db8f7a5ad91f02b6a2323c10a4187","No archived body","https://moderna.com/llms.txt","https://moderna.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"228","biogen.com","Biogen","Healthcare & pharma","Biotech","United States","Global","2026-10-21","https://www.biogen.com/robots.txt","200","ok","346","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","4","has_strict_links","4","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/biogen.com.txt","03f8452b6804bec83ad83f5d54e150cd06db762c697711e5ea1fad45ab0f7b85","03f8452b6804bec83ad83f5d54e150cd06db762c697711e5ea1fad45ab0f7b85","Matches source index","https://biogen.com/robots.txt","https://www.biogen.com/robots.txt","llms/biogen.com.txt","758c992cd48cc4c03948aee77a0b30ecfd2c4f27d974b0603466fc2d155bc6f5","758c992cd48cc4c03948aee77a0b30ecfd2c4f27d974b0603466fc2d155bc6f5","Matches source index","https://biogen.com/llms.txt","https://www.biogen.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/biogen.com.txt","llms/biogen.com.txt","03f8452b6804bec83ad83f5d54e150cd06db762c697711e5ea1fad45ab0f7b85","758c992cd48cc4c03948aee77a0b30ecfd2c4f27d974b0603466fc2d155bc6f5","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"229","regeneron.com","Regeneron","Healthcare & pharma","Biotech","United States","Global","2026-10-21","https://www.regeneron.com/robots.txt","200","ok","2345","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/regeneron.com.txt","5615d305c5c551f46407eb6d55d20409c84e961024369ca00b4b0c568d1e0ea5","5615d305c5c551f46407eb6d55d20409c84e961024369ca00b4b0c568d1e0ea5","Matches source index","https://regeneron.com/robots.txt","https://www.regeneron.com/robots.txt","","","d1e9b27dea9d11b66430d71511c3390b693cb0c31647baba00aca151a4e3e843","No archived body","https://regeneron.com/llms.txt","https://www.regeneron.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/regeneron.com.txt","","5615d305c5c551f46407eb6d55d20409c84e961024369ca00b4b0c568d1e0ea5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"230","mayoclinic.org","Mayo Clinic","Healthcare & pharma","Health system","United States","Global","2026-10-22","https://mayoclinic.org/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://mayoclinic.org/robots.txt","https://mayoclinic.org/robots.txt","","","3f85cfec9212e428e90bc0b4105a7d6b49627f6364e3c71736f3611390668daa","No archived body","https://mayoclinic.org/llms.txt","https://www.mayoclinic.org/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-b0b7d9f3aaec","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-82e568e5010c","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"231","clevelandclinic.org","Cleveland Clinic","Healthcare & pharma","Health system","United States","Global","2026-10-21","https://my.clevelandclinic.org/robots.txt","200","ok","1064","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","10","has_strict_links","10","True","robots.txt redirects to host my.clevelandclinic.org","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/clevelandclinic.org.txt","a194b4ba24f77365a0a9dc31272ca4fab4b7efaa7f1a44e25c89ac2f79712419","a194b4ba24f77365a0a9dc31272ca4fab4b7efaa7f1a44e25c89ac2f79712419","Matches source index","https://clevelandclinic.org/robots.txt","https://my.clevelandclinic.org/robots.txt","llms/clevelandclinic.org.txt","d88f9e49c3ea2473f1299fc57c8ba3ed2e0ee0f43051b8bf6469edd232e410b5","d88f9e49c3ea2473f1299fc57c8ba3ed2e0ee0f43051b8bf6469edd232e410b5","Matches source index","https://clevelandclinic.org/llms.txt","https://www.clevelandclinic.org/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/clevelandclinic.org.txt","llms/clevelandclinic.org.txt","a194b4ba24f77365a0a9dc31272ca4fab4b7efaa7f1a44e25c89ac2f79712419","d88f9e49c3ea2473f1299fc57c8ba3ed2e0ee0f43051b8bf6469edd232e410b5","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"232","hopkinsmedicine.org","Johns Hopkins Medicine","Healthcare & pharma","Health system","United States","Global","2026-10-21","https://www.hopkinsmedicine.org/robots.txt","200","ok","407","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hopkinsmedicine.org.txt","6a7bc185e69595db24514656e74f7c5a20eaef83d4994574be25c59a86db0888","6a7bc185e69595db24514656e74f7c5a20eaef83d4994574be25c59a86db0888","Matches source index","https://hopkinsmedicine.org/robots.txt","https://www.hopkinsmedicine.org/robots.txt","","","","No stored body; provider HTTP response in task record","https://hopkinsmedicine.org/llms.txt","https://www.hopkinsmedicine.org/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/hopkinsmedicine.org.txt","","6a7bc185e69595db24514656e74f7c5a20eaef83d4994574be25c59a86db0888","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-b0c60441047a","2026-10-22 23:43:00 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"233","kaiserpermanente.org","Kaiser Permanente","Healthcare & pharma","Health system","United States","Global","2026-10-21","https://healthy.kaiserpermanente.org/robots.txt?kp_shortcut_referrer=kp.org/robots.txt","200","ok","6653","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host healthy.kaiserpermanente.org","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/kaiserpermanente.org.txt","f2dc41c15121080a6eafd049146131bb6802f237ea1e22a9d52af6e42321c70a","f2dc41c15121080a6eafd049146131bb6802f237ea1e22a9d52af6e42321c70a","Matches source index","https://kaiserpermanente.org/robots.txt","https://healthy.kaiserpermanente.org/robots.txt?kp_shortcut_referrer=kp.org/robots.txt","llms/kaiserpermanente.org.txt","847fc5d3fd110fb35285c66114e6252cb80e115547694c9aa8bd1385030aeb40","847fc5d3fd110fb35285c66114e6252cb80e115547694c9aa8bd1385030aeb40","Matches source index","https://kaiserpermanente.org/llms.txt","https://healthy.kaiserpermanente.org/front-door","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/kaiserpermanente.org.txt","llms/kaiserpermanente.org.txt","f2dc41c15121080a6eafd049146131bb6802f237ea1e22a9d52af6e42321c70a","847fc5d3fd110fb35285c66114e6252cb80e115547694c9aa8bd1385030aeb40","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"234","hcahealthcare.com","HCA Healthcare","Healthcare & pharma","Health system","United States","Global","2026-10-21","https://www.hcahealthcare.com/robots.txt","200","ok","5381","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hcahealthcare.com.txt","4fc1b58f632a01e8df0a029f67f25fdf566f0a0dff52eac9cb35a1f7c2bde6e0","4fc1b58f632a01e8df0a029f67f25fdf566f0a0dff52eac9cb35a1f7c2bde6e0","Matches source index","https://hcahealthcare.com/robots.txt","https://www.hcahealthcare.com/robots.txt","llms/hcahealthcare.com.txt","3aa76efd1ad5e4686a116cd9efe11c706927997e973d4ab90d684caedf5e6e55","3aa76efd1ad5e4686a116cd9efe11c706927997e973d4ab90d684caedf5e6e55","Matches source index","https://hcahealthcare.com/llms.txt","https://geo-blocked-site.dpxmedcity.net","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hcahealthcare.com.txt","llms/hcahealthcare.com.txt","4fc1b58f632a01e8df0a029f67f25fdf566f0a0dff52eac9cb35a1f7c2bde6e0","3aa76efd1ad5e4686a116cd9efe11c706927997e973d4ab90d684caedf5e6e55","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"235","cvshealth.com","CVS Health","Healthcare & pharma","Healthcare","United States","Global","2026-10-21","https://www.cvshealth.com/robots.txt","200","ok","228","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","42","True","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/cvshealth.com.txt","c4a13eba56d767a9bb3df05a64a2db7b03a74980edfc3ec8e281eff2901e39a4","c4a13eba56d767a9bb3df05a64a2db7b03a74980edfc3ec8e281eff2901e39a4","Matches source index","https://cvshealth.com/robots.txt","https://www.cvshealth.com/robots.txt","crawler-api-verification-pass/raw/llms/cvshealth.com--10222342-2533-0275-0000-34038f5622e0.txt","c7ef5479c129155686513014b6e96c6d32fec8b3e7f0058117f13285615d673d","","API-returned UTF-8 body; byte length matched provider size","https://cvshealth.com/llms.txt","https://www.cvshealth.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/cvshealth.com.txt","","c4a13eba56d767a9bb3df05a64a2db7b03a74980edfc3ec8e281eff2901e39a4","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222342-2533-0275-0000-34038f5622e0","2026-10-22 23:42:58 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-03.json","1"
"236","unitedhealthgroup.com","UnitedHealth Group","Healthcare & pharma","Health insurer","United States","Global","2026-10-21","https://unitedhealthgroup.com/robots.txt","200","ok","7325","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/unitedhealthgroup.com.txt","19198ea4ca83ced3b2ecb1a37ebd4c0842c62f08df5c41d7cce290694217346e","19198ea4ca83ced3b2ecb1a37ebd4c0842c62f08df5c41d7cce290694217346e","Matches source index","https://unitedhealthgroup.com/robots.txt","https://unitedhealthgroup.com/robots.txt","llms/unitedhealthgroup.com.txt","b2511847ebc9f2b5b85595651bbd24cc1f7b09530dee2a7b9c282ef4c568db25","b2511847ebc9f2b5b85595651bbd24cc1f7b09530dee2a7b9c282ef4c568db25","Matches source index","https://unitedhealthgroup.com/llms.txt","https://www.unitedhealthgroup.com/page-not-found.html","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/unitedhealthgroup.com.txt","llms/unitedhealthgroup.com.txt","19198ea4ca83ced3b2ecb1a37ebd4c0842c62f08df5c41d7cce290694217346e","b2511847ebc9f2b5b85595651bbd24cc1f7b09530dee2a7b9c282ef4c568db25","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"237","webmd.com","WebMD","Healthcare & pharma","Health publisher","United States","Global","2026-10-22","https://www.webmd.com/robots.txt","200","ok","886","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/webmd.com.txt","e7610682d0934ea2b4266992e2a88b3ca70c2e2ff9f8047223b03c2ca4fbce7a","e7610682d0934ea2b4266992e2a88b3ca70c2e2ff9f8047223b03c2ca4fbce7a","Matches source index","https://webmd.com/robots.txt","https://www.webmd.com/robots.txt","","","96300ab460eef6cc2740fa7c0e20c81b0e27f4f5d2246809ffef6b6f64c9b4b7","No archived body","https://webmd.com/llms.txt","https://www.webmd.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/webmd.com.txt","","e7610682d0934ea2b4266992e2a88b3ca70c2e2ff9f8047223b03c2ca4fbce7a","","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"238","healthline.com","Healthline","Healthcare & pharma","Health publisher","United States","Global","2026-10-21","https://www.healthline.com/robots.txt","200","ok","8864","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/healthline.com.txt","bdd5879797eabcb955749cdf9e754cb3b23510de793835ba5588e3f91e87130a","bdd5879797eabcb955749cdf9e754cb3b23510de793835ba5588e3f91e87130a","Matches source index","https://healthline.com/robots.txt","https://www.healthline.com/robots.txt","","","a9200315db715216f9b434b8d07d707e8373c519a02196b5eb76a1dc200dc89d","No archived body","https://healthline.com/llms.txt","https://www.healthline.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/healthline.com.txt","","bdd5879797eabcb955749cdf9e754cb3b23510de793835ba5588e3f91e87130a","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"239","medicalnewstoday.com","Medical News Today","Healthcare & pharma","Health publisher","","Global","2026-10-21","https://www.medicalnewstoday.com/robots.txt","200","ok","1713","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/medicalnewstoday.com.txt","314c4a8ebc9b7a037ee074ff507f2986d05ee40f6060e4f6590ac1b20b4dde38","314c4a8ebc9b7a037ee074ff507f2986d05ee40f6060e4f6590ac1b20b4dde38","Matches source index","https://medicalnewstoday.com/robots.txt","https://www.medicalnewstoday.com/robots.txt","","","fa77905a73420b2c03007c20225b3bc288e096fd67bc2b444c307a667d19d1c3","No archived body","https://medicalnewstoday.com/llms.txt","https://www.medicalnewstoday.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/medicalnewstoday.com.txt","","314c4a8ebc9b7a037ee074ff507f2986d05ee40f6060e4f6590ac1b20b4dde38","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"240","nhs.uk","NHS","Healthcare & pharma","Health authority","United Kingdom","Global","2026-10-21","https://www.nhs.uk/robots.txt","200","ok","2674","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nhs.uk.txt","ee6b60406a7277008cdd3bdedba97ca6c3ade405f269b0656ac704598e960ed0","ee6b60406a7277008cdd3bdedba97ca6c3ade405f269b0656ac704598e960ed0","Matches source index","https://nhs.uk/robots.txt","https://www.nhs.uk/robots.txt","","","a1593aff15d752101bc4e1d2f55b6f5de6f8034b093c6c79fb9488873baa34e4","No archived body","https://nhs.uk/llms.txt","https://www.nhs.uk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nhs.uk.txt","","ee6b60406a7277008cdd3bdedba97ca6c3ade405f269b0656ac704598e960ed0","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"241","mit.edu","Massachusetts Institute of Technology","Education & research","University","United States","Global","2026-10-21","https://web.mit.edu/robots.txt","200","ok","199","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host web.mit.edu","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mit.edu.txt","bad4e8ead1b0414e6d9f15a570ae195b73c69f2fd02e3e2516e2991289675a47","bad4e8ead1b0414e6d9f15a570ae195b73c69f2fd02e3e2516e2991289675a47","Matches source index","https://mit.edu/robots.txt","https://web.mit.edu/robots.txt","","","168c3da82cb9b3f68dcdf04c29761893dd113ebfb16a9e87c21698d681459c21","No archived body","https://mit.edu/llms.txt","https://web.mit.edu/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mit.edu.txt","","bad4e8ead1b0414e6d9f15a570ae195b73c69f2fd02e3e2516e2991289675a47","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"242","stanford.edu","Stanford University","Education & research","University","United States","Global","2026-10-21","https://www.stanford.edu/robots.txt","200","ok","197","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/stanford.edu.txt","aadf777d4727200060490d80c1e1c942e45fbdc03ee179aaee40ef1de72dffbd","aadf777d4727200060490d80c1e1c942e45fbdc03ee179aaee40ef1de72dffbd","Matches source index","https://stanford.edu/robots.txt","https://www.stanford.edu/robots.txt","","","dfa4261e9af3c600a80dbbf3e55b063c3ad3eb707259e166e648e9eb04bce4a2","No archived body","https://stanford.edu/llms.txt","https://www.stanford.edu/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/stanford.edu.txt","","aadf777d4727200060490d80c1e1c942e45fbdc03ee179aaee40ef1de72dffbd","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"243","harvard.edu","Harvard University","Education & research","University","United States","Global","2026-10-21","https://www.harvard.edu/robots.txt","200","ok","748","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/harvard.edu.txt","14353df5d44dabd47535f2f91e1fbfdf3e0cc8de10bfe8f5d79e11b1d9cc2474","14353df5d44dabd47535f2f91e1fbfdf3e0cc8de10bfe8f5d79e11b1d9cc2474","Matches source index","https://harvard.edu/robots.txt","https://www.harvard.edu/robots.txt","","","d2bdbc31bd3a944fbc0d7cec738279f999e97d60b303185b9a24887f80776ca3","No archived body","https://harvard.edu/llms.txt","https://www.harvard.edu/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/harvard.edu.txt","","14353df5d44dabd47535f2f91e1fbfdf3e0cc8de10bfe8f5d79e11b1d9cc2474","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"244","ox.ac.uk","University of Oxford","Education & research","University","United Kingdom","Global","2026-10-21","https://www.ox.ac.uk/robots.txt","200","ok","2121","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ox.ac.uk.txt","06a2f016f42aff0ec26a4a2da5d193f0fbfaf68071885c7f173f943ca4f5a8f5","06a2f016f42aff0ec26a4a2da5d193f0fbfaf68071885c7f173f943ca4f5a8f5","Matches source index","https://ox.ac.uk/robots.txt","https://www.ox.ac.uk/robots.txt","","","6263b69a0dbac9d46b043d3cd2174458fa1df3e92e428b28b5f12c1f723135f3","No archived body","https://www.ox.ac.uk/llms.txt","https://www.ox.ac.uk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ox.ac.uk.txt","","06a2f016f42aff0ec26a4a2da5d193f0fbfaf68071885c7f173f943ca4f5a8f5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"245","cam.ac.uk","University of Cambridge","Education & research","University","United Kingdom","Global","2026-10-21","https://www.cam.ac.uk/robots.txt","200","ok","2027","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cam.ac.uk.txt","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","Matches source index","https://cam.ac.uk/robots.txt","https://www.cam.ac.uk/robots.txt","","","7ca433b0d78ee35c39215dcdf2eb8cac323e6703bfcd2b37334f37243327a9c0","No archived body","https://cam.ac.uk/llms.txt","https://www.cam.ac.uk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cam.ac.uk.txt","","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"246","caltech.edu","California Institute of Technology","Education & research","University","United States","Global","2026-10-22","https://www.caltech.edu/robots.txt","200","ok","265","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","edu', port=443): Max retries exceeded with url: /","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/caltech.edu.txt","74387dc126c53b39a5582ed56e0ca96b1dd807a41a4b952b06476ffb63032771","74387dc126c53b39a5582ed56e0ca96b1dd807a41a4b952b06476ffb63032771","Matches source index","https://caltech.edu/robots.txt","https://www.caltech.edu/robots.txt","","","5bced1a17a0fd54076a27200f57c896144e215eb697f72cf8b49bc812c77e1ae","No archived body","https://www.caltech.edu/llms.txt","https://www.caltech.edu/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/caltech.edu.txt","","74387dc126c53b39a5582ed56e0ca96b1dd807a41a4b952b06476ffb63032771","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"247","princeton.edu","Princeton University","Education & research","University","United States","Global","2026-10-21","https://www.Princeton.EDU/robots.txt","200","ok","2275","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/princeton.edu.txt","9052eb9806da2eec4c0c24ff91feb640a55e6f91b8f584180450b6007931ebae","9052eb9806da2eec4c0c24ff91feb640a55e6f91b8f584180450b6007931ebae","Matches source index","https://princeton.edu/robots.txt","https://www.Princeton.EDU/robots.txt","","","463e3c7fbdc7884de697b5ad7aaabfde0dca67e895b4125bf33aff3d5f95eef8","No archived body","https://princeton.edu/llms.txt","https://www.Princeton.EDU/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/princeton.edu.txt","","9052eb9806da2eec4c0c24ff91feb640a55e6f91b8f584180450b6007931ebae","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"248","yale.edu","Yale University","Education & research","University","United States","Global","2026-10-21","https://www.yale.edu/robots.txt","200","ok","2491","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/yale.edu.txt","1fee25e01aa5853418d99cc376de585f2d7e1f85acdf3e880ffb187f68f30878","1fee25e01aa5853418d99cc376de585f2d7e1f85acdf3e880ffb187f68f30878","Matches source index","https://yale.edu/robots.txt","https://www.yale.edu/robots.txt","","","a9d6c63fbbfd2c42e9949f8ad26b5bdb829a08a062763886bdb07495f5015905","No archived body","https://yale.edu/llms.txt","https://www.yale.edu/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/yale.edu.txt","","1fee25e01aa5853418d99cc376de585f2d7e1f85acdf3e880ffb187f68f30878","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"249","columbia.edu","Columbia University","Education & research","University","United States","Global","2026-10-21","https://columbia.edu/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://columbia.edu/robots.txt","https://columbia.edu/robots.txt","","","b393f255068c31973165153a76deacf9f9cbe73edbcfc160dad8c18fea107799","No archived body","https://columbia.edu/llms.txt","https://columbia.edu/llms.txt","404","text/html; charset=iso-8859-1","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"250","uchicago.edu","University of Chicago","Education & research","University","United States","Global","2026-10-21","https://www.uchicago.edu/robots.txt","200","ok","243","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/uchicago.edu.txt","da6c5a82ed98ddc747d809dc8507b49247173ac5020df5b6127ae54abf45fd37","da6c5a82ed98ddc747d809dc8507b49247173ac5020df5b6127ae54abf45fd37","Matches source index","https://uchicago.edu/robots.txt","https://www.uchicago.edu/robots.txt","","","90b2d35cd5e08370ed20db81197dd9da1a4dbb421f71293fd5733ea49eb7b3e1","No archived body","https://uchicago.edu/llms.txt","https://www.uchicago.edu/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/uchicago.edu.txt","","da6c5a82ed98ddc747d809dc8507b49247173ac5020df5b6127ae54abf45fd37","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"251","imperial.ac.uk","Imperial College London","Education & research","University","United Kingdom","Global","2026-10-21","https://www.imperial.ac.uk/robots.txt","200","ok","12713","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","21","has_strict_links","21","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/imperial.ac.uk.txt","a36a6736f9eb67a32e771fd5348fc24707926cba96301962bbbe75887c9a2f8c","a36a6736f9eb67a32e771fd5348fc24707926cba96301962bbbe75887c9a2f8c","Matches source index","https://imperial.ac.uk/robots.txt","https://www.imperial.ac.uk/robots.txt","llms/imperial.ac.uk.txt","db2cb425142395e5c9b2219e74b864a491709b04ab53eefae5e70af5aa94d6d2","db2cb425142395e5c9b2219e74b864a491709b04ab53eefae5e70af5aa94d6d2","Matches source index","https://imperial.ac.uk/llms.txt","https://www.imperial.ac.uk/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/imperial.ac.uk.txt","llms/imperial.ac.uk.txt","a36a6736f9eb67a32e771fd5348fc24707926cba96301962bbbe75887c9a2f8c","db2cb425142395e5c9b2219e74b864a491709b04ab53eefae5e70af5aa94d6d2","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"252","ucl.ac.uk","University College London","Education & research","University","United Kingdom","Global","2026-10-22","https://www.ucl.ac.uk/robots.txt","200","ok","18458","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/ucl.ac.uk.txt","9330e00edf72db57f83ec22c0e936caecb3e4f3e63e83b413093240860881989","9330e00edf72db57f83ec22c0e936caecb3e4f3e63e83b413093240860881989","Matches source index","https://ucl.ac.uk/robots.txt","https://www.ucl.ac.uk/robots.txt","","","ac6543445e3ec4985bc0e26bf2708560d5cde28c56e13a40db802a8cdeb534fb","No archived body","https://ucl.ac.uk/llms.txt","https://www.ucl.ac.uk/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","unverified","Unverified","raw/ucl.ac.uk.txt","","9330e00edf72db57f83ec22c0e936caecb3e4f3e63e83b413093240860881989","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222347-2533-0275-0000-8ca0d55fffed","2026-10-22 23:47:22 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-pilot.json","2"
"253","ethz.ch","ETH Zurich","Education & research","University","Switzerland","Global","2026-10-21","https://ethz.ch/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://ethz.ch/robots.txt","https://ethz.ch/robots.txt","","","7f1b4bb58af9d9da3e6e6c212c9b2ab47048fed4ac054f215f73660b7e62b6d1","No archived body","https://ethz.ch/llms.txt","https://ethz.ch/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"254","tsinghua.edu.cn","Tsinghua University","Education & research","University","China","Global","2026-10-22","https://www.tsinghua.edu.cn/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed). edu.cn', port=443): Max retries exceeded with url: /edu.","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://tsinghua.edu.cn/robots.txt","https://www.tsinghua.edu.cn/robots.txt","","","29c0b0a5db57af93122bca27d4c301f5d1b7d34e5f410a85b4c1af2e747a1407","No archived body","https://www.tsinghua.edu.cn/llms.txt","https://www.tsinghua.edu.cn/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-22","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"255","utoronto.ca","University of Toronto","Education & research","University","Canada","Global","2026-10-21","https://www.utoronto.ca/robots.txt","200","ok","2096","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","29","has_strict_links","29","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/utoronto.ca.txt","bcc081773a225230c2a3a524edc83c5962fbb1f33027ae7003e78197e3f5b4a4","bcc081773a225230c2a3a524edc83c5962fbb1f33027ae7003e78197e3f5b4a4","Matches source index","https://utoronto.ca/robots.txt","https://www.utoronto.ca/robots.txt","llms/utoronto.ca.txt","8e78f2e8c49e0e9f344100472923909883f1e7633c4b5c65d99f564d42839a21","8e78f2e8c49e0e9f344100472923909883f1e7633c4b5c65d99f564d42839a21","Matches source index","https://utoronto.ca/llms.txt","https://www.utoronto.ca/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/utoronto.ca.txt","llms/utoronto.ca.txt","bcc081773a225230c2a3a524edc83c5962fbb1f33027ae7003e78197e3f5b4a4","8e78f2e8c49e0e9f344100472923909883f1e7633c4b5c65d99f564d42839a21","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"256","mcgill.ca","McGill University","Education & research","University","Canada","Global","2026-10-22","https://mcgill.ca/robots.txt","200","ok","3386","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mcgill.ca.txt","34318cefd2355fc215ff30781bdbf60b401a49910c8b3903cb4b097bce7a8be4","34318cefd2355fc215ff30781bdbf60b401a49910c8b3903cb4b097bce7a8be4","Matches source index","https://mcgill.ca/robots.txt","https://mcgill.ca/robots.txt","","","481381b5b2aa76853e92039b55e214c885844abea70f76b2ab02d2dbd8ce8763","No archived body","https://mcgill.ca/llms.txt","https://www.mcgill.ca/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/mcgill.ca.txt","","34318cefd2355fc215ff30781bdbf60b401a49910c8b3903cb4b097bce7a8be4","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"257","nature.com","Nature","Education & research","Scholarly publisher","","Global","2026-10-21","https://www.nature.com/robots.txt","200","ok","2780","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nature.com.txt","542ab5d887e3f7171534cae96934bf64b7a1ad4226be398169aa9ca006da4521","542ab5d887e3f7171534cae96934bf64b7a1ad4226be398169aa9ca006da4521","Matches source index","https://nature.com/robots.txt","https://www.nature.com/robots.txt","","","e5534cb4c66fb5f5b7d199549edc8fe4972300733fe013c8e0aee68280e0ccf3","No archived body","https://nature.com/llms.txt","https://www.nature.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nature.com.txt","","542ab5d887e3f7171534cae96934bf64b7a1ad4226be398169aa9ca006da4521","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"258","arxiv.org","arXiv","Education & research","Preprint repository","United States","Global","2026-10-21","https://arxiv.org/robots.txt","200","ok","5854","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/arxiv.org.txt","eda2bc852443f3f29d31eb04d37b26476654dc33b29256fb882819da23d189a5","eda2bc852443f3f29d31eb04d37b26476654dc33b29256fb882819da23d189a5","Matches source index","https://arxiv.org/robots.txt","https://arxiv.org/robots.txt","","","c46fa63f7f0b64a1e171733b98bd283182c43e5eb76525a58435fc621e45c74c","No archived body","https://arxiv.org/llms.txt","https://arxiv.org/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/arxiv.org.txt","","eda2bc852443f3f29d31eb04d37b26476654dc33b29256fb882819da23d189a5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"259","science.org","Science","Education & research","Scholarly publisher","United States","Global","2026-10-21","https://www.science.org/robots.txt","200","ok","930","valid_txt_200","True","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/science.org.txt","b18ab37ce2c1095bca3f8d962fddb7a00752bba380fddfdd541b1c2d26a92400","b18ab37ce2c1095bca3f8d962fddb7a00752bba380fddfdd541b1c2d26a92400","Matches source index","https://science.org/robots.txt","https://www.science.org/robots.txt","","","c33533855c845469954c9b91f51be3bc5e44dfe2bdd7d9ebe85591be84312517","No archived body","https://science.org/llms.txt","https://www.science.org/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/science.org.txt","","b18ab37ce2c1095bca3f8d962fddb7a00752bba380fddfdd541b1c2d26a92400","","Blocked","Partial","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-ab33d4500dc8","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"260","plos.org","PLOS","Education & research","Scholarly publisher","United States","Global","2026-10-21","https://plos.org/robots.txt","200","ok","166","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/plos.org.txt","22bea5a80f3c1e5a295ab6812377491ec416750e147a57c38ea55514321c2bd2","22bea5a80f3c1e5a295ab6812377491ec416750e147a57c38ea55514321c2bd2","Matches source index","https://plos.org/robots.txt","https://plos.org/robots.txt","","","96a3e311f2a2bee0bddde64c75955990b094cd6379a663d20fa4a45020cb424a","No archived body","https://plos.org/llms.txt","https://plos.org/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/plos.org.txt","","22bea5a80f3c1e5a295ab6812377491ec416750e147a57c38ea55514321c2bd2","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"261","springer.com","Springer Nature","Education & research","Scholarly publisher","Germany","Global","2026-10-21","https://www.springer.com/robots.txt","200","ok","286","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/springer.com.txt","9deaa339bbddb3b20c37c05b7537188f86204eb966c6fd98b8eac31cd406e696","9deaa339bbddb3b20c37c05b7537188f86204eb966c6fd98b8eac31cd406e696","Matches source index","https://springer.com/robots.txt","https://www.springer.com/robots.txt","llms/springer.com.txt","32ed63159c77e21ee19ca1b9aa3213ccf0218eb59539560b132a8e68ef0e18ea","32ed63159c77e21ee19ca1b9aa3213ccf0218eb59539560b132a8e68ef0e18ea","Matches source index","https://www.springer.com/llms.txt","https://link.springer.com/brands/springer","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/springer.com.txt","llms/springer.com.txt","9deaa339bbddb3b20c37c05b7537188f86204eb966c6fd98b8eac31cd406e696","32ed63159c77e21ee19ca1b9aa3213ccf0218eb59539560b132a8e68ef0e18ea","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"262","elsevier.com","Elsevier","Education & research","Scholarly publisher","Netherlands","Global","2026-10-21","https://www.elsevier.com/robots.txt","200","ok","490","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/elsevier.com.txt","4e72b2758f3b723f9a5f92d4e046da7e20348246acf3cf9c4b97c840fe9f3533","4e72b2758f3b723f9a5f92d4e046da7e20348246acf3cf9c4b97c840fe9f3533","Matches source index","https://elsevier.com/robots.txt","https://www.elsevier.com/robots.txt","","","c1fe3264142044364f70a4953d27041e2a1f8cf5b01d47ce7935e3db2cb9e716","No archived body","https://elsevier.com/llms.txt","https://www.elsevier.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/elsevier.com.txt","","4e72b2758f3b723f9a5f92d4e046da7e20348246acf3cf9c4b97c840fe9f3533","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"263","wiley.com","Wiley","Education & research","Scholarly publisher","United States","Global","2026-10-22","https://www.wiley.com/robots.txt","200","ok","1014","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wiley.com.txt","b0f714de0e2a5f41534c14bfa66b743a55866d4c0cc9f88b6b41744b722fbfb8","b0f714de0e2a5f41534c14bfa66b743a55866d4c0cc9f88b6b41744b722fbfb8","Matches source index","https://wiley.com/robots.txt","https://www.wiley.com/robots.txt","llms/wiley.com.txt","55a38cbf132467dea9fabbb8327e417f55e2cbe44b52829ce0a35c97d7fa2d9d","55a38cbf132467dea9fabbb8327e417f55e2cbe44b52829ce0a35c97d7fa2d9d","Matches source index","https://wiley.com/llms.txt","https://www.wiley.com/en-us","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/wiley.com.txt","llms/wiley.com.txt","b0f714de0e2a5f41534c14bfa66b743a55866d4c0cc9f88b6b41744b722fbfb8","55a38cbf132467dea9fabbb8327e417f55e2cbe44b52829ce0a35c97d7fa2d9d","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"264","researchgate.net","ResearchGate","Education & research","Scholarly platform","Germany","Global","2026-10-21","https://www.researchgate.net/robots.txt","200","ok","293","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/researchgate.net.txt","4b9963df3f3529b6ee2b812decf2dadfde6d5eb041f547401dcf4915ac4bf1cb","4b9963df3f3529b6ee2b812decf2dadfde6d5eb041f547401dcf4915ac4bf1cb","Matches source index","https://researchgate.net/robots.txt","https://www.researchgate.net/robots.txt","","","","No stored body; provider HTTP response in task record","https://researchgate.net/llms.txt","https://www.researchgate.net/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/researchgate.net.txt","","4b9963df3f3529b6ee2b812decf2dadfde6d5eb041f547401dcf4915ac4bf1cb","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-ea6e4a3a8842","2026-10-22 23:42:58 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"265","academia.edu","Academia.edu","Education & research","Scholarly platform","United States","Global","2026-10-21","https://www.academia.edu/robots.txt","200","ok","59291","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","True","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/academia.edu.txt","850b27d82470c0ebb698d9a28585607490acf8494c4752a1861e3e184506a7da","850b27d82470c0ebb698d9a28585607490acf8494c4752a1861e3e184506a7da","Matches source index","https://academia.edu/robots.txt","https://www.academia.edu/robots.txt","","","f18aa6f839900a746c76ba076a9afd1e32252c74b69755288b26ced878d1122f","No archived body","https://academia.edu/llms.txt","https://www.academia.edu/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/academia.edu.txt","","850b27d82470c0ebb698d9a28585607490acf8494c4752a1861e3e184506a7da","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"266","coursera.org","Coursera","Education & research","Education platform","United States","Global","2026-10-21","https://www.coursera.org/robots.txt","200","ok","5345","valid_txt_200","True","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","45","has_strict_links","45","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/coursera.org.txt","11565c6ec6224646f5d75cfb3911f28edca51860e3499dd633f19b11011becda","11565c6ec6224646f5d75cfb3911f28edca51860e3499dd633f19b11011becda","Matches source index","https://coursera.org/robots.txt","https://www.coursera.org/robots.txt","llms/coursera.org.txt","6ead111dd6d6f3dfd861afb7bfc3498995557898ef8f2509950d1beaaf21c6ce","6ead111dd6d6f3dfd861afb7bfc3498995557898ef8f2509950d1beaaf21c6ce","Matches source index","https://coursera.org/llms.txt","https://www.coursera.org/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/coursera.org.txt","llms/coursera.org.txt","11565c6ec6224646f5d75cfb3911f28edca51860e3499dd633f19b11011becda","6ead111dd6d6f3dfd861afb7bfc3498995557898ef8f2509950d1beaaf21c6ce","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"267","edx.org","edX","Education & research","Education platform","United States","Global","2026-10-21","https://www.edx.org/robots.txt","200","ok","1346","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","40","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/edx.org.txt","c1c682ca6478f4b3b83ac763efc1d494c1363a200a5c06acdf22b00a0722c315","c1c682ca6478f4b3b83ac763efc1d494c1363a200a5c06acdf22b00a0722c315","Matches source index","https://edx.org/robots.txt","https://www.edx.org/robots.txt","llms/edx.org.txt","bfc9d033e7025fc9db292ec74a29a7b28e18ab7a2bc9813ebf42388f5ce14451","bfc9d033e7025fc9db292ec74a29a7b28e18ab7a2bc9813ebf42388f5ce14451","Matches source index","https://edx.org/llms.txt","https://www.edx.org/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/edx.org.txt","llms/edx.org.txt","c1c682ca6478f4b3b83ac763efc1d494c1363a200a5c06acdf22b00a0722c315","bfc9d033e7025fc9db292ec74a29a7b28e18ab7a2bc9813ebf42388f5ce14451","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"268","khanacademy.org","Khan Academy","Education & research","Education platform","United States","Global","2026-10-22","https://www.khanacademy.org/robots.txt","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/khanacademy.org.txt","4965d403ea8006362b7b9527e254092d06a584fa5335eddaca51255af1efe08e","","Computed from archived invalid body; source index hash absent","https://khanacademy.org/robots.txt","https://www.khanacademy.org/robots.txt","llms/khanacademy.org.txt","4965d403ea8006362b7b9527e254092d06a584fa5335eddaca51255af1efe08e","4965d403ea8006362b7b9527e254092d06a584fa5335eddaca51255af1efe08e","Matches source index","https://khanacademy.org/llms.txt","https://www.khanacademy.org/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/khanacademy.org.txt","llms/khanacademy.org.txt","4965d403ea8006362b7b9527e254092d06a584fa5335eddaca51255af1efe08e","4965d403ea8006362b7b9527e254092d06a584fa5335eddaca51255af1efe08e","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222349-2533-0275-0000-2b8b414f6686","2026-10-22 23:49:45 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","","","","","","",""
"269","mpg.de","Max Planck Society","Education & research","Research institute","Germany","Global","2026-10-21","https://www.mpg.de/robots.txt","200","ok","87","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mpg.de.txt","cc97aa00089a847ab5d3855e35393b2bfbacb2a208ac9793bc9a4069229a7a3c","cc97aa00089a847ab5d3855e35393b2bfbacb2a208ac9793bc9a4069229a7a3c","Matches source index","https://mpg.de/robots.txt","https://www.mpg.de/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://mpg.de/llms.txt","https://www.mpg.de/llms.txt","404","text/plain; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mpg.de.txt","","cc97aa00089a847ab5d3855e35393b2bfbacb2a208ac9793bc9a4069229a7a3c","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"270","cern.ch","CERN","Education & research","Research institute","International","Global","2026-10-21","https://cern.ch/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://cern.ch/robots.txt","https://cern.ch/robots.txt","","","4a35381341f79a710b3ef66525cb9ad31543a6fb170ace866754c1890bcf0b7e","No archived body","https://cern.ch/llms.txt","https://cern.ch/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"271","booking.com","Booking.com","Travel, hospitality & airlines","Booking platform","Netherlands","Global","2026-10-21","https://booking.com/robots.txt","200","ok","39491","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/booking.com.txt","d4d6f0840e2ed2b056ab44353eee55b1c8ddbbe2be3c0e82c4c942b0e93bfe6b","d4d6f0840e2ed2b056ab44353eee55b1c8ddbbe2be3c0e82c4c942b0e93bfe6b","Matches source index","https://booking.com/robots.txt","https://booking.com/robots.txt","","","6bdef6fd4480a1088ec0e200d50842895a5ff79036356b2d98c539260a1369f5","No archived body","https://booking.com/llms.txt","https://www.booking.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/booking.com.txt","","d4d6f0840e2ed2b056ab44353eee55b1c8ddbbe2be3c0e82c4c942b0e93bfe6b","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"272","expedia.com","Expedia","Travel, hospitality & airlines","Booking platform","United States","Global","2026-10-21","https://www.expedia.com/robots.txt","200","ok","2210","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","15","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/expedia.com.txt","3d5c2166f871be025f083a9a3aa6895e8dad4e6c4d2f5e8a24c809d11c0b7cac","3d5c2166f871be025f083a9a3aa6895e8dad4e6c4d2f5e8a24c809d11c0b7cac","Matches source index","https://expedia.com/robots.txt","https://www.expedia.com/robots.txt","llms/expedia.com.txt","0257a8ed950141a03755cbdb98e3851909df6eaa72a985c2416a1db413e82978","0257a8ed950141a03755cbdb98e3851909df6eaa72a985c2416a1db413e82978","Matches source index","https://expedia.com/llms.txt","https://www.expedia.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/expedia.com.txt","llms/expedia.com.txt","3d5c2166f871be025f083a9a3aa6895e8dad4e6c4d2f5e8a24c809d11c0b7cac","0257a8ed950141a03755cbdb98e3851909df6eaa72a985c2416a1db413e82978","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"273","airbnb.com","Airbnb","Travel, hospitality & airlines","Booking platform","United States","Global","2026-10-21","https://www.airbnb.com/robots.txt","200","ok","20930","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/airbnb.com.txt","e274a840ea29686b39867202b958149e3097a57bd608136ef09565e84c049fb2","e274a840ea29686b39867202b958149e3097a57bd608136ef09565e84c049fb2","Matches source index","https://airbnb.com/robots.txt","https://www.airbnb.com/robots.txt","","","a71bb7d237aea979f1174a535454c4447a1c804e6f4a8f4574679906eef5d85a","No archived body","https://airbnb.com/llms.txt","https://www.airbnb.com/404","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/airbnb.com.txt","","e274a840ea29686b39867202b958149e3097a57bd608136ef09565e84c049fb2","","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"274","marriott.com","Marriott","Travel, hospitality & airlines","Hotel group","United States","Global","2026-10-21","https://www.marriott.com/robots.txt","200","ok","2643","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/marriott.com.txt","3fde12219521507a4bd08941b71a20c72c84f13b2d64cafbcb01adeb2c76bed8","3fde12219521507a4bd08941b71a20c72c84f13b2d64cafbcb01adeb2c76bed8","Matches source index","https://marriott.com/robots.txt","https://www.marriott.com/robots.txt","","","4e9014983ba097f3c9c2dd26a412ada301c59d1c73e7ec5d076b091d73325426","No archived body","https://marriott.com/llms.txt","https://www.marriott.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/marriott.com.txt","","3fde12219521507a4bd08941b71a20c72c84f13b2d64cafbcb01adeb2c76bed8","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"275","hilton.com","Hilton","Travel, hospitality & airlines","Hotel group","United States","Global","2026-10-21","https://www.hilton.com/robots.txt","200","ok","770","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hilton.com.txt","7dcfdd1c709ea42fe582aa4d5a72810208e7207bf9ad5acff48a57b4ab35ceec","7dcfdd1c709ea42fe582aa4d5a72810208e7207bf9ad5acff48a57b4ab35ceec","Matches source index","https://hilton.com/robots.txt","https://www.hilton.com/robots.txt","","","0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","No archived body","https://hilton.com/llms.txt","https://www.hilton.com/llms.txt","404","text/html; charset=iso-8859-1","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hilton.com.txt","","7dcfdd1c709ea42fe582aa4d5a72810208e7207bf9ad5acff48a57b4ab35ceec","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"276","hyatt.com","Hyatt","Travel, hospitality & airlines","Hotel group","United States","Global","2026-10-22","https://hyatt.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://hyatt.com/robots.txt","https://hyatt.com/robots.txt","","","796434fc509b0468cc27293551676745c267e5155155b78149fe8b9c49c24f5a","No archived body","https://hyatt.com/llms.txt","https://www.hyatt.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-e4aeeab95b95","2026-10-22 23:49:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-fb584efa88f2","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"277","ihg.com","IHG Hotels & Resorts","Travel, hospitality & airlines","Hotel group","United Kingdom","Global","2026-10-22","https://ihg.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://ihg.com/robots.txt","https://ihg.com/robots.txt","","","49864c53e10b7794eb51951c51f950cc68c09fcd105917fa7c0ba651faff41bb","No archived body","https://ihg.com/llms.txt","https://ihg.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222349-2533-0275-0000-973569108818","2026-10-22 23:49:44 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-87fc18d3a779","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"278","wyndhamhotels.com","Wyndham Hotels & Resorts","Travel, hospitality & airlines","Hotel group","United States","Global","2026-10-21","https://www.wyndhamhotels.com/robots.txt","200","ok","1680","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","86","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/wyndhamhotels.com.txt","0f8ed92a024114a39288af05ee7ae1edcc106611f42b0b831699e1e7af3011ab","0f8ed92a024114a39288af05ee7ae1edcc106611f42b0b831699e1e7af3011ab","Matches source index","https://wyndhamhotels.com/robots.txt","https://www.wyndhamhotels.com/robots.txt","llms/wyndhamhotels.com.txt","562cdb7174319c274a2507fecd140f14b14af23aaee6f3ba95635206e149da60","562cdb7174319c274a2507fecd140f14b14af23aaee6f3ba95635206e149da60","Matches source index","https://wyndhamhotels.com/llms.txt","https://www.wyndhamhotels.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/wyndhamhotels.com.txt","llms/wyndhamhotels.com.txt","0f8ed92a024114a39288af05ee7ae1edcc106611f42b0b831699e1e7af3011ab","562cdb7174319c274a2507fecd140f14b14af23aaee6f3ba95635206e149da60","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"279","delta.com","Delta Air Lines","Travel, hospitality & airlines","Airline","United States","Global","2026-10-22","https://www.delta.com/robots.txt","200","ok","6163","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Max retries exceeded with url: /com', port=443","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/delta.com.txt","017c295a41dc8886391a98cd9e7f4988d95def0d99315a0f1945510ee19950db","017c295a41dc8886391a98cd9e7f4988d95def0d99315a0f1945510ee19950db","Matches source index","https://delta.com/robots.txt","https://www.delta.com/robots.txt","","","f55bdec55e0a1df1e6f51a459c36a35cde178e3b0c025daabaa417b531440242","No archived body","https://www.delta.com/llms.txt","https://www.delta.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/delta.com.txt","","017c295a41dc8886391a98cd9e7f4988d95def0d99315a0f1945510ee19950db","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"280","united.com","United Airlines","Travel, hospitality & airlines","Airline","United States","Global","2026-10-22","https://united.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.united.com', port=80): Read timed out. (read timeout=20) united.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://united.com/robots.txt","https://united.com/robots.txt","","","","No archived body","https://united.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222349-2533-0275-0000-83c5e61be03d","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-robots-01.json","2","Frozen v3.1.2 baseline","Undetermined","","10222351-2533-0275-0000-a4e4ebef26c1","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"281","aa.com","American Airlines","Travel, hospitality & airlines","Airline","United States","Global","2026-10-21","https://www.aa.com/robots.txt","200","ok","587","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aa.com.txt","a3fa6e76667ee984d31b2ab0c7009bb673448c7f619bd8c86e1135d9bbff3e5c","a3fa6e76667ee984d31b2ab0c7009bb673448c7f619bd8c86e1135d9bbff3e5c","Matches source index","https://aa.com/robots.txt","https://www.aa.com/robots.txt","","","4ffc23fa48bcac34d16ff18d162efcb59a933788d65919d00585be9d0b9e10e5","No archived body","https://aa.com/llms.txt","https://www.aa.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aa.com.txt","","a3fa6e76667ee984d31b2ab0c7009bb673448c7f619bd8c86e1135d9bbff3e5c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"282","southwest.com","Southwest Airlines","Travel, hospitality & airlines","Airline","United States","Global","2026-10-21","https://www.southwest.com/robots.txt","200","ok","531","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/southwest.com.txt","27daadae0b135e2fec78c4390714bb1c1f4b35a495648fe0b4183ec6e2787383","27daadae0b135e2fec78c4390714bb1c1f4b35a495648fe0b4183ec6e2787383","Matches source index","https://southwest.com/robots.txt","https://www.southwest.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://southwest.com/llms.txt","https://www.southwest.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/southwest.com.txt","","27daadae0b135e2fec78c4390714bb1c1f4b35a495648fe0b4183ec6e2787383","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-1f6a0851cbfa","2026-10-22 23:43:01 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"283","jetblue.com","JetBlue","Travel, hospitality & airlines","Airline","United States","Global","2026-10-21","https://www.jetblue.com/robots.txt","200","ok","255","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 406 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jetblue.com.txt","66dc19140d5163c691c510a0e6369bc6891b936e6aaf9cf9a770de2675749ab9","66dc19140d5163c691c510a0e6369bc6891b936e6aaf9cf9a770de2675749ab9","Matches source index","https://jetblue.com/robots.txt","https://www.jetblue.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://jetblue.com/llms.txt","https://www.jetblue.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/jetblue.com.txt","","66dc19140d5163c691c510a0e6369bc6891b936e6aaf9cf9a770de2675749ab9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222342-2533-0275-0000-430cd38a4ab9","2026-10-22 23:42:58 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-03.json","1"
"284","alaskaair.com","Alaska Airlines","Travel, hospitality & airlines","Airline","United States","Global","2026-10-21","https://www.alaskaair.com/robots.txt","200","ok","1965","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","other","0","no_strict_links","587","True","Baseline note:  llms.txt: HTTP 406 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/alaskaair.com.txt","303366f41d0a4710b78ce67fa770b2acca3742d811dc169d4078e74f9716cd06","303366f41d0a4710b78ce67fa770b2acca3742d811dc169d4078e74f9716cd06","Matches source index","https://alaskaair.com/robots.txt","https://www.alaskaair.com/robots.txt","crawler-api-verification-pass/raw/llms/alaskaair.com--10222351-2533-0275-0000-af82b05d16d8.txt","a57f3ccf82045115ceae05cf404c49139e4f7991e62c81aa93d7e2512035fe80","","API-returned UTF-8 body; byte length matched provider size","https://alaskaair.com/llms.txt","https://www.alaskaair.com/content/page-not-found","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/alaskaair.com.txt","","303366f41d0a4710b78ce67fa770b2acca3742d811dc169d4078e74f9716cd06","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","200","10222351-2533-0275-0000-af82b05d16d8","2026-10-22 23:52:00 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"285","britishairways.com","British Airways","Travel, hospitality & airlines","Airline","United Kingdom","Global","2026-10-22","http://britishairways.com/robots.txt","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/britishairways.com.txt","55d59f6b1f0c0277cd4862e584034186179d79baba9f305e8186e7c27966c946","","Computed from archived invalid body; source index hash absent","https://britishairways.com/robots.txt","http://britishairways.com/robots.txt","llms/britishairways.com.txt","f71a8dbc120ff800cdecb20232391fe6c0833e235261ced8cc1280c48da65a05","f71a8dbc120ff800cdecb20232391fe6c0833e235261ced8cc1280c48da65a05","Matches source index","http://britishairways.com/llms.txt","http://britishairways.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/britishairways.com.txt","llms/britishairways.com.txt","55d59f6b1f0c0277cd4862e584034186179d79baba9f305e8186e7c27966c946","f71a8dbc120ff800cdecb20232391fe6c0833e235261ced8cc1280c48da65a05","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222350-2533-0275-0000-3dda7764c33c","2026-10-22 23:50:45 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"286","lufthansa.com","Lufthansa","Travel, hospitality & airlines","Airline","Germany","Global","2026-10-21","https://www.lufthansa.com/robots.txt","200","ok","242","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","comment_banner_other","0","no_strict_links","7","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/lufthansa.com.txt","1f58c4a3d770cccce246495d9eb15fba60a668239ce549b4d1555be1db141e1e","1f58c4a3d770cccce246495d9eb15fba60a668239ce549b4d1555be1db141e1e","Matches source index","https://lufthansa.com/robots.txt","https://www.lufthansa.com/robots.txt","llms/lufthansa.com.txt","967047ea2b878239a89491bcfc6314480e022184cc39a9a8e5e9a9f0bb8a88bf","967047ea2b878239a89491bcfc6314480e022184cc39a9a8e5e9a9f0bb8a88bf","Matches source index","https://lufthansa.com/llms.txt","https://www.lufthansa.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/lufthansa.com.txt","llms/lufthansa.com.txt","1f58c4a3d770cccce246495d9eb15fba60a668239ce549b4d1555be1db141e1e","967047ea2b878239a89491bcfc6314480e022184cc39a9a8e5e9a9f0bb8a88bf","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"287","airfrance.com","Air France","Travel, hospitality & airlines","Airline","France","Global","2026-10-22","https://airfrance.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.airfrance.com', port=80): Read timed out. (read timeout=20) airfrance.com', port=443): Max retries exceeded with url: /airfr llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://airfrance.com/robots.txt","https://airfrance.com/robots.txt","","","","No archived body","https://airfrance.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222350-2533-0275-0000-92dac992f49f","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","","10222351-2533-0275-0000-7d84b6238f18","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"288","klm.com","KLM","Travel, hospitality & airlines","Airline","Netherlands","Global","2026-10-21","https://www.klm.com/robots.txt","200","ok","2779","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/klm.com.txt","ebcd99c5b7f4b95581ecfb99ef3a972080c6a225c2c3303640cc3e8aef5cfdeb","ebcd99c5b7f4b95581ecfb99ef3a972080c6a225c2c3303640cc3e8aef5cfdeb","Matches source index","https://klm.com/robots.txt","https://www.klm.com/robots.txt","","","5aca67ae8ea15bb70999a8b82949b138db748fa3c305e316d408795cc678947a","No archived body","https://klm.com/llms.txt","https://www.klm.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/klm.com.txt","","ebcd99c5b7f4b95581ecfb99ef3a972080c6a225c2c3303640cc3e8aef5cfdeb","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"289","ryanair.com","Ryanair","Travel, hospitality & airlines","Airline","Ireland","Global","2026-10-22","https://www.ryanair.com/robots.txt","200","ok","717","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","ryanair.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ryanair.com.txt","477ac5d451d277244649fd240201b1c109b04b2923e1550b566b4607a1c20679","477ac5d451d277244649fd240201b1c109b04b2923e1550b566b4607a1c20679","Matches source index","https://ryanair.com/robots.txt","https://www.ryanair.com/robots.txt","llms/ryanair.com.txt","f4b7330459ba480d9f33aa4fd8b5c75f4a8b59b1e3fee2298b4116ddc90c8cf9","f4b7330459ba480d9f33aa4fd8b5c75f4a8b59b1e3fee2298b4116ddc90c8cf9","Matches source index","https://ryanair.com/llms.txt","https://www.ryanair.com/ie/en","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/ryanair.com.txt","llms/ryanair.com.txt","477ac5d451d277244649fd240201b1c109b04b2923e1550b566b4607a1c20679","f4b7330459ba480d9f33aa4fd8b5c75f4a8b59b1e3fee2298b4116ddc90c8cf9","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"290","easyjet.com","easyJet","Travel, hospitality & airlines","Airline","United Kingdom","Global","2026-10-22","https://easyjet.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://easyjet.com/robots.txt","https://easyjet.com/robots.txt","","","673de591f3b75b6ef05cb16a503965ca8f4b2f80d1ad951b374e409ead06cda4","No archived body","https://easyjet.com/llms.txt","https://www.easyjet.com","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-0b0e98c6200e","2026-10-22 23:50:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-9fcc6a79d154","2026-10-22 23:51:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"291","virginatlantic.com","Virgin Atlantic","Travel, hospitality & airlines","Airline","United Kingdom","Global","2026-10-21","https://www.virginatlantic.com/robots.txt","200","ok","669","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/virginatlantic.com.txt","f40190ab89c669c8342f47fab5c811c4d95597b0d2797710da97bf7961c4f5cd","f40190ab89c669c8342f47fab5c811c4d95597b0d2797710da97bf7961c4f5cd","Matches source index","https://virginatlantic.com/robots.txt","https://www.virginatlantic.com/robots.txt","","","8bd77b8dec2357a161f15f1e6284bdf8f3e065147fc5bf40de3f12fb956d5744","No archived body","https://virginatlantic.com/llms.txt","https://www.virginatlantic.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/virginatlantic.com.txt","","f40190ab89c669c8342f47fab5c811c4d95597b0d2797710da97bf7961c4f5cd","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"292","qantas.com","Qantas","Travel, hospitality & airlines","Airline","Australia","Global","2026-10-21","https://www.qantas.com/robots.txt","200","ok","1616","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/qantas.com.txt","d0bcb86b3bff473609feec03b2f700534c9fbf6877c1ad32670d81b3c814c3ef","d0bcb86b3bff473609feec03b2f700534c9fbf6877c1ad32670d81b3c814c3ef","Matches source index","https://qantas.com/robots.txt","https://www.qantas.com/robots.txt","","","4a69cf21553377d16f0f1f88d02d209428fb84dd927e52234fc47e9bc252f909","No archived body","https://qantas.com/llms.txt","https://www.qantas.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/qantas.com.txt","","d0bcb86b3bff473609feec03b2f700534c9fbf6877c1ad32670d81b3c814c3ef","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"293","singaporeair.com","Singapore Airlines","Travel, hospitality & airlines","Airline","Singapore","Global","2026-10-21","https://www.singaporeair.com/robots.txt","200","ok","1194","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/singaporeair.com.txt","838e8c1298cbe46e1ce431e08b2958f5464c38590b4972f755623b224ebb3438","838e8c1298cbe46e1ce431e08b2958f5464c38590b4972f755623b224ebb3438","Matches source index","https://singaporeair.com/robots.txt","https://www.singaporeair.com/robots.txt","","","33078c3a6062e2bb9805a7b4575707d6332bf1b7c4dd78522be340e4fd0318b6","No archived body","https://singaporeair.com/llms.txt","https://www.singaporeair.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/singaporeair.com.txt","","838e8c1298cbe46e1ce431e08b2958f5464c38590b4972f755623b224ebb3438","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"294","cathaypacific.com","Cathay Pacific","Travel, hospitality & airlines","Airline","Hong Kong","Global","2026-10-21","https://www.cathaypacific.com/robots.txt","200","ok","3183","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cathaypacific.com.txt","35db4692ca4958d399bed312424b70032582987c84aa9d79ba42c46e37c98ed0","35db4692ca4958d399bed312424b70032582987c84aa9d79ba42c46e37c98ed0","Matches source index","https://cathaypacific.com/robots.txt","https://www.cathaypacific.com/robots.txt","","","4cf0e17d922e5625068d839b4274ad6c53488464cf5d8aae7f4f4c5664f4e446","No archived body","https://cathaypacific.com/llms.txt","https://www.cathaypacific.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cathaypacific.com.txt","","35db4692ca4958d399bed312424b70032582987c84aa9d79ba42c46e37c98ed0","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"295","ana.co.jp","ANA - All Nippon Airways","Travel, hospitality & airlines","Airline","Japan","Global","2026-10-21","https://www.ana.co.jp/robots.txt","200","ok","116","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","co.jp', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ana.co.jp.txt","252a0e5c62d4c4981261d53f55b1d986c8d94777218fe8bf68cd5af042cd3c5e","252a0e5c62d4c4981261d53f55b1d986c8d94777218fe8bf68cd5af042cd3c5e","Matches source index","https://ana.co.jp/robots.txt","https://www.ana.co.jp/robots.txt","","","f59c3a27e4531f0e145da8bcb20f9dd160bda9d3d63de42a723cd6c9f454649b","No archived body","https://ana.co.jp/llms.txt","https://www.ana.co.jp/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ana.co.jp.txt","","252a0e5c62d4c4981261d53f55b1d986c8d94777218fe8bf68cd5af042cd3c5e","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"296","jal.co.jp","Japan Airlines","Travel, hospitality & airlines","Airline","Japan","Global","2026-10-22","https://jal.co.jp/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://jal.co.jp/robots.txt","https://jal.co.jp/robots.txt","","","7b490cbacc595576d4e256d6db964f94a48b540ba84082f0b7c4aa6db767e78c","No archived body","https://jal.co.jp/llms.txt","https://www.jal.co.jp/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-6f668af095e9","2026-10-22 23:50:47 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-f11bcd9af1e4","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"297","trip.com","Trip.com","Travel, hospitality & airlines","Booking platform","","Global","2026-10-21","https://www.trip.com/robots.txt","200","ok","4587","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","196","has_strict_links","223","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/trip.com.txt","8d7336d2b21d2896e3765f463052ccda38d02d2c19d6f8df9d8ef203d492c967","8d7336d2b21d2896e3765f463052ccda38d02d2c19d6f8df9d8ef203d492c967","Matches source index","https://trip.com/robots.txt","https://www.trip.com/robots.txt","llms/trip.com.txt","e36bb43cfac86555334606c3e5111faaf9b52138f2ce22081755a7d65cd71a14","e36bb43cfac86555334606c3e5111faaf9b52138f2ce22081755a7d65cd71a14","Matches source index","https://trip.com/llms.txt","https://www.trip.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/trip.com.txt","llms/trip.com.txt","8d7336d2b21d2896e3765f463052ccda38d02d2c19d6f8df9d8ef203d492c967","e36bb43cfac86555334606c3e5111faaf9b52138f2ce22081755a7d65cd71a14","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"298","agoda.com","Agoda","Travel, hospitality & airlines","Booking platform","","Global","2026-10-21","https://www.agoda.com/robots.txt","200","ok","5086","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/agoda.com.txt","c47b9fe96c3eb0eb5298b6787dea4b21f97f1616831d9399bf7a7525b68af873","c47b9fe96c3eb0eb5298b6787dea4b21f97f1616831d9399bf7a7525b68af873","Matches source index","https://agoda.com/robots.txt","https://www.agoda.com/robots.txt","","","c01b62e49a61ea893d402810141be3f5ab2dcaa916133009aae7de4ae8112bde","No archived body","https://agoda.com/llms.txt","https://www.agoda.com/pagenotfound.html","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/agoda.com.txt","","c47b9fe96c3eb0eb5298b6787dea4b21f97f1616831d9399bf7a7525b68af873","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"299","hotels.com","Hotels.com","Travel, hospitality & airlines","Booking platform","United States","Global","2026-10-21","https://www.hotels.com/robots.txt","200","ok","2522","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/hotels.com.txt","98144f27918421b34e91b8352479ad7949e2a268333e22cfeaca723941317010","98144f27918421b34e91b8352479ad7949e2a268333e22cfeaca723941317010","Matches source index","https://hotels.com/robots.txt","https://www.hotels.com/robots.txt","","","7324fd482fcf82831cc65517b355066f74f572044c6460d6d8608f9e11280adc","No archived body","https://hotels.com/llms.txt","https://www.hotels.com/llms.txt","403","application/xml","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/hotels.com.txt","","98144f27918421b34e91b8352479ad7949e2a268333e22cfeaca723941317010","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222351-2533-0275-0000-c3ac4203e3e2","2026-10-22 23:51:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-02.json","2"
"300","vrbo.com","Vrbo","Travel, hospitality & airlines","Booking platform","United States","Global","2026-10-21","https://www.vrbo.com/robots.txt","200","ok","2046","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/vrbo.com.txt","e1b81859b9181b92d8885244583dcf99602ea0b2303cda3a3084696fcb8f3e29","e1b81859b9181b92d8885244583dcf99602ea0b2303cda3a3084696fcb8f3e29","Matches source index","https://vrbo.com/robots.txt","https://www.vrbo.com/robots.txt","","","d46f2c9f7501071cfd02815ec8dce7c13622577c0cf1aeaf0e7ed733104044ca","No archived body","https://vrbo.com/llms.txt","https://www.vrbo.com/llms.txt","403","application/xml","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/vrbo.com.txt","","e1b81859b9181b92d8885244583dcf99602ea0b2303cda3a3084696fcb8f3e29","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-225482d562c9","2026-10-22 23:52:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"301","searchengineland.com","Search Engine Land","Marketing, SEO & media industry","SEO publication","United States","Global","2026-10-21","https://searchengineland.com/robots.txt","200","ok","170","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","5504","has_strict_links","5504","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/searchengineland.com.txt","18f5ec361216ef8fc2e27acacc6cf44f74a48cc8b699d1012fd0a3907e310bba","18f5ec361216ef8fc2e27acacc6cf44f74a48cc8b699d1012fd0a3907e310bba","Matches source index","https://searchengineland.com/robots.txt","https://searchengineland.com/robots.txt","llms/searchengineland.com.txt","760a26e5be03e845500a78bcd7a091df430728338b6fc86239ec1aac04ba1892","760a26e5be03e845500a78bcd7a091df430728338b6fc86239ec1aac04ba1892","Matches source index","https://searchengineland.com/llms.txt","https://searchengineland.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/searchengineland.com.txt","llms/searchengineland.com.txt","18f5ec361216ef8fc2e27acacc6cf44f74a48cc8b699d1012fd0a3907e310bba","760a26e5be03e845500a78bcd7a091df430728338b6fc86239ec1aac04ba1892","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"302","searchenginejournal.com","Search Engine Journal","Marketing, SEO & media industry","SEO publication","United States","Global","2026-10-21","https://www.searchenginejournal.com/robots.txt","200","ok","2020","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","141","has_strict_links","141","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/searchenginejournal.com.txt","d71c57076ed7bd98b9b451181669572e6d2ca8874c8013a58aaad2410dbb521e","d71c57076ed7bd98b9b451181669572e6d2ca8874c8013a58aaad2410dbb521e","Matches source index","https://searchenginejournal.com/robots.txt","https://www.searchenginejournal.com/robots.txt","llms/searchenginejournal.com.txt","09ba02005aa40e88377f4055b6714729775be9602df73d5ba098c6636ea08a05","09ba02005aa40e88377f4055b6714729775be9602df73d5ba098c6636ea08a05","Matches source index","https://searchenginejournal.com/llms.txt","https://www.searchenginejournal.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/searchenginejournal.com.txt","llms/searchenginejournal.com.txt","d71c57076ed7bd98b9b451181669572e6d2ca8874c8013a58aaad2410dbb521e","09ba02005aa40e88377f4055b6714729775be9602df73d5ba098c6636ea08a05","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"303","moz.com","Moz","Marketing, SEO & media industry","SEO tool","United States","Global","2026-10-21","https://moz.com/robots.txt","200","ok","1207","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/moz.com.txt","79bb8772fcda6f8a96ff58fb7f53975fc39d3cc98f48091b69f6510f5ea9e8fa","79bb8772fcda6f8a96ff58fb7f53975fc39d3cc98f48091b69f6510f5ea9e8fa","Matches source index","https://moz.com/robots.txt","https://moz.com/robots.txt","","","ef5cd8c5d101cd92578c511741b80cf5036757ad1c5f36fcb3f087d869fdb22d","No archived body","https://moz.com/llms.txt","https://moz.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/moz.com.txt","","79bb8772fcda6f8a96ff58fb7f53975fc39d3cc98f48091b69f6510f5ea9e8fa","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"304","ahrefs.com","Ahrefs","Marketing, SEO & media industry","SEO tool","Singapore","Global","2026-10-21","https://ahrefs.com/robots.txt","200","ok","908","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ahrefs.com.txt","01fc1026b5da28c13524ab1efefe89329c010e5ea9f37df55719d4aa38880da8","01fc1026b5da28c13524ab1efefe89329c010e5ea9f37df55719d4aa38880da8","Matches source index","https://ahrefs.com/robots.txt","https://ahrefs.com/robots.txt","","","d465172175d35d493fb1633e237700022bd849fa123164790b168b8318acb090","No archived body","https://ahrefs.com/llms.txt","https://ahrefs.com/llms.txt","404","text/html; charset=utf8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ahrefs.com.txt","","01fc1026b5da28c13524ab1efefe89329c010e5ea9f37df55719d4aa38880da8","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"305","semrush.com","Semrush","Marketing, SEO & media industry","SEO tool","United States","Global","2026-10-21","https://www.semrush.com/robots.txt","200","ok","3599","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","42","has_strict_links","42","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/semrush.com.txt","65bac391e0e7a745709be15387ee0539f1153ebe893b67aa3a43ef1270b70a1a","65bac391e0e7a745709be15387ee0539f1153ebe893b67aa3a43ef1270b70a1a","Matches source index","https://semrush.com/robots.txt","https://www.semrush.com/robots.txt","llms/semrush.com.txt","49efaf655823299eeaa37259c0161d42b2c927b94bb68036092aff0a4497755e","49efaf655823299eeaa37259c0161d42b2c927b94bb68036092aff0a4497755e","Matches source index","https://semrush.com/llms.txt","https://www.semrush.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/semrush.com.txt","llms/semrush.com.txt","65bac391e0e7a745709be15387ee0539f1153ebe893b67aa3a43ef1270b70a1a","49efaf655823299eeaa37259c0161d42b2c927b94bb68036092aff0a4497755e","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"306","hubspot.com","HubSpot","Marketing, SEO & media industry","Marketing platform","United States","Global","2026-10-21","https://www.hubspot.com/robots.txt","200","ok","12606","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","225","has_strict_links","232","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/hubspot.com.txt","74374bc3165f346bbb5959ad5d2948e2e16f1a364ba8ccb49663cb14eaae96af","74374bc3165f346bbb5959ad5d2948e2e16f1a364ba8ccb49663cb14eaae96af","Matches source index","https://hubspot.com/robots.txt","https://www.hubspot.com/robots.txt","llms/hubspot.com.txt","e1d66e610875c2e7f320de1c026b9fb8c49759d7670bdbd318630ae7d536c491","e1d66e610875c2e7f320de1c026b9fb8c49759d7670bdbd318630ae7d536c491","Matches source index","https://hubspot.com/llms.txt","https://www.hubspot.com/llms.txt","200","text/plain; charset=x-macroman","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/hubspot.com.txt","llms/hubspot.com.txt","74374bc3165f346bbb5959ad5d2948e2e16f1a364ba8ccb49663cb14eaae96af","e1d66e610875c2e7f320de1c026b9fb8c49759d7670bdbd318630ae7d536c491","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"307","neilpatel.com","Neil Patel","Marketing, SEO & media industry","Marketing publication","United States","Global","2026-10-21","https://neilpatel.com/robots.txt","200","ok","573","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","78","has_strict_links","78","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/neilpatel.com.txt","cee8b90e369c0a6034ecd69604506683979fefa685f9ca8575b041ccffd43018","cee8b90e369c0a6034ecd69604506683979fefa685f9ca8575b041ccffd43018","Matches source index","https://neilpatel.com/robots.txt","https://neilpatel.com/robots.txt","llms/neilpatel.com.txt","8592d6676bffe5eca4d456a1f12bbdcc5f30f4abea373d51999ad7c008080e26","8592d6676bffe5eca4d456a1f12bbdcc5f30f4abea373d51999ad7c008080e26","Matches source index","https://neilpatel.com/llms.txt","https://neilpatel.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/neilpatel.com.txt","llms/neilpatel.com.txt","cee8b90e369c0a6034ecd69604506683979fefa685f9ca8575b041ccffd43018","8592d6676bffe5eca4d456a1f12bbdcc5f30f4abea373d51999ad7c008080e26","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"308","backlinko.com","Backlinko","Marketing, SEO & media industry","SEO publication","United States","Global","2026-10-21","https://backlinko.com/robots.txt","200","ok","418","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/backlinko.com.txt","70f503a382992e1e8e040d3e571a873e508210f7cdc612a65ecce43451110cc7","70f503a382992e1e8e040d3e571a873e508210f7cdc612a65ecce43451110cc7","Matches source index","https://backlinko.com/robots.txt","https://backlinko.com/robots.txt","","","184cfe95d8f1d31e0b0199be62c5cba1158d61923e53ad7cceb0ff2ab7f3da2a","No archived body","https://backlinko.com/llms.txt","https://backlinko.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/backlinko.com.txt","","70f503a382992e1e8e040d3e571a873e508210f7cdc612a65ecce43451110cc7","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"309","contentmarketinginstitute.com","Content Marketing Institute","Marketing, SEO & media industry","Marketing publication","United States","Global","2026-10-21","https://contentmarketinginstitute.com/robots.txt","200","ok","1525","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/contentmarketinginstitute.com.txt","a76c261ecdd63c984528cacbd90deae88b194407ddcc820846aabcda6a1fd655","a76c261ecdd63c984528cacbd90deae88b194407ddcc820846aabcda6a1fd655","Matches source index","https://contentmarketinginstitute.com/robots.txt","https://contentmarketinginstitute.com/robots.txt","","","ccd6277759cb151826ca7efa5f34717d322be078eb95d1281d277e5e9448a0ee","No archived body","https://contentmarketinginstitute.com/llms.txt","https://contentmarketinginstitute.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/contentmarketinginstitute.com.txt","","a76c261ecdd63c984528cacbd90deae88b194407ddcc820846aabcda6a1fd655","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"310","marketingprofs.com","MarketingProfs","Marketing, SEO & media industry","Marketing publication","United States","Global","2026-10-21","https://www.marketingprofs.com/robots.txt","200","ok","1091","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/marketingprofs.com.txt","f606a811974403a3cd80cc9ebf5678f81cec2a4645adf15db5a0f4ae72f02272","f606a811974403a3cd80cc9ebf5678f81cec2a4645adf15db5a0f4ae72f02272","Matches source index","https://marketingprofs.com/robots.txt","https://www.marketingprofs.com/robots.txt","","","3ea6fc1b542ae4748a400ef3f3996b3addbf67392d05989fc62f30814fc2bd34","No archived body","https://marketingprofs.com/llms.txt","https://www.marketingprofs.com/llms.txt","404","text/html; Charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/marketingprofs.com.txt","","f606a811974403a3cd80cc9ebf5678f81cec2a4645adf15db5a0f4ae72f02272","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"311","socialmediaexaminer.com","Social Media Examiner","Marketing, SEO & media industry","Marketing publication","United States","Global","2026-10-21","https://socialmediaexaminer.com/robots.txt","200","ok","2438","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","11","has_strict_links","11","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/socialmediaexaminer.com.txt","e03fb789902816a9e76420b73778877dd496c2c38d4d9bb26aae2493d5729edc","e03fb789902816a9e76420b73778877dd496c2c38d4d9bb26aae2493d5729edc","Matches source index","https://socialmediaexaminer.com/robots.txt","https://socialmediaexaminer.com/robots.txt","llms/socialmediaexaminer.com.txt","65ef609910183f33564b9b0753973ebbe3e12d7c17ea909478202b5f5480b0ff","65ef609910183f33564b9b0753973ebbe3e12d7c17ea909478202b5f5480b0ff","Matches source index","https://socialmediaexaminer.com/llms.txt","https://socialmediaexaminer.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/socialmediaexaminer.com.txt","llms/socialmediaexaminer.com.txt","e03fb789902816a9e76420b73778877dd496c2c38d4d9bb26aae2493d5729edc","65ef609910183f33564b9b0753973ebbe3e12d7c17ea909478202b5f5480b0ff","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"312","buffer.com","Buffer","Marketing, SEO & media industry","Social media tool","United States","Global","2026-10-21","https://buffer.com/robots.txt","200","ok","498","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","63","has_strict_links","64","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/buffer.com.txt","524e2b574ce6dedf5ea38817693a575bce6b9521ea5fa12b5feadd9f51140ec7","524e2b574ce6dedf5ea38817693a575bce6b9521ea5fa12b5feadd9f51140ec7","Matches source index","https://buffer.com/robots.txt","https://buffer.com/robots.txt","llms/buffer.com.txt","c038ca4438974a01b69debbbd806fad5ae90ca511e5fdb69639e23fca942fa52","c038ca4438974a01b69debbbd806fad5ae90ca511e5fdb69639e23fca942fa52","Matches source index","https://buffer.com/llms.txt","https://developers.buffer.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/buffer.com.txt","llms/buffer.com.txt","524e2b574ce6dedf5ea38817693a575bce6b9521ea5fa12b5feadd9f51140ec7","c038ca4438974a01b69debbbd806fad5ae90ca511e5fdb69639e23fca942fa52","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"313","hootsuite.com","Hootsuite","Marketing, SEO & media industry","Social media tool","Canada","Global","2026-10-21","https://www.hootsuite.com/robots.txt","200","ok","7994","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","100","has_strict_links","100","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/hootsuite.com.txt","1ecbd3d26f07369cba914bc5c1d4f1f43b2ea9aeffdcf4f27871f99bf32fb726","1ecbd3d26f07369cba914bc5c1d4f1f43b2ea9aeffdcf4f27871f99bf32fb726","Matches source index","https://hootsuite.com/robots.txt","https://www.hootsuite.com/robots.txt","llms/hootsuite.com.txt","e8dfb513c040bef26582f21d4bb8ebdd7ad6fe1cf99f77ce1c661b54d0aa1b37","e8dfb513c040bef26582f21d4bb8ebdd7ad6fe1cf99f77ce1c661b54d0aa1b37","Matches source index","https://hootsuite.com/llms.txt","https://www.hootsuite.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/hootsuite.com.txt","llms/hootsuite.com.txt","1ecbd3d26f07369cba914bc5c1d4f1f43b2ea9aeffdcf4f27871f99bf32fb726","e8dfb513c040bef26582f21d4bb8ebdd7ad6fe1cf99f77ce1c661b54d0aa1b37","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"314","sproutsocial.com","Sprout Social","Marketing, SEO & media industry","Social media tool","United States","Global","2026-10-21","https://sproutsocial.com/robots.txt","200","ok","383","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","100","has_strict_links","100","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/sproutsocial.com.txt","c796aa3577b0e858b615ab39e8616c3e42f2dc3b25323cd8e48c5a585edb4fc6","c796aa3577b0e858b615ab39e8616c3e42f2dc3b25323cd8e48c5a585edb4fc6","Matches source index","https://sproutsocial.com/robots.txt","https://sproutsocial.com/robots.txt","llms/sproutsocial.com.txt","c93be44ffb31b831db5190f06482526b99c6d0a7a6f7a38b92d6cebe489f0aa2","c93be44ffb31b831db5190f06482526b99c6d0a7a6f7a38b92d6cebe489f0aa2","Matches source index","https://sproutsocial.com/llms.txt","https://sproutsocial.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/sproutsocial.com.txt","llms/sproutsocial.com.txt","c796aa3577b0e858b615ab39e8616c3e42f2dc3b25323cd8e48c5a585edb4fc6","c93be44ffb31b831db5190f06482526b99c6d0a7a6f7a38b92d6cebe489f0aa2","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"315","copyblogger.com","Copyblogger","Marketing, SEO & media industry","Content marketing publication","United States","Global","2026-10-21","https://copyblogger.com/robots.txt","200","ok","156","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/copyblogger.com.txt","b2385f54624d3efead9d859c0223b74e0b0fc18af0df87de4620724620adecac","b2385f54624d3efead9d859c0223b74e0b0fc18af0df87de4620724620adecac","Matches source index","https://copyblogger.com/robots.txt","https://copyblogger.com/robots.txt","","","c36c89337ad69997f6f56403b128caf5f4d0958d7cba356e6990d340f9f99d31","No archived body","https://copyblogger.com/llms.txt","https://copyblogger.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/copyblogger.com.txt","","b2385f54624d3efead9d859c0223b74e0b0fc18af0df87de4620724620adecac","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"316","convinceandconvert.com","Convince & Convert","Marketing, SEO & media industry","Marketing agency & publication","United States","Global","2026-10-22","https://convinceandconvert.com/robots.txt/","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/convinceandconvert.com.txt","faf0e750e3fac90c8a83d8296839075b2cef0342b678a44d9d17d2745ebdd5c4","","Computed from archived invalid body; source index hash absent","https://convinceandconvert.com/robots.txt","https://convinceandconvert.com/robots.txt/","llms/convinceandconvert.com.txt","b65dd1b1f03b5961ff861ce5f5af6502d1bf758efe07b25117a07f918d05c227","b65dd1b1f03b5961ff861ce5f5af6502d1bf758efe07b25117a07f918d05c227","Matches source index","https://convinceandconvert.com/llms.txt","https://convinceandconvert.com/llms.txt/","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/convinceandconvert.com.txt","llms/convinceandconvert.com.txt","faf0e750e3fac90c8a83d8296839075b2cef0342b678a44d9d17d2745ebdd5c4","b65dd1b1f03b5961ff861ce5f5af6502d1bf758efe07b25117a07f918d05c227","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222350-2533-0275-0000-2b975f8954f1","2026-10-22 23:50:46 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"317","adweek.com","Adweek","Marketing, SEO & media industry","Advertising publication","United States","Global","2026-10-21","https://www.adweek.com/robots.txt","200","ok","9250","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","False","True","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/adweek.com.txt","9bfe15680ccb942c3db074f0993528a8ba75231a0c44892626c431e92e3c24e9","9bfe15680ccb942c3db074f0993528a8ba75231a0c44892626c431e92e3c24e9","Matches source index","https://adweek.com/robots.txt","https://www.adweek.com/robots.txt","","","23934b6cd2e5299cb83fa40b0522098cbb3be9acf33ad5a88801bd45344322a4","No archived body","https://adweek.com/llms.txt","https://www.adweek.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/adweek.com.txt","","9bfe15680ccb942c3db074f0993528a8ba75231a0c44892626c431e92e3c24e9","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"318","adage.com","Ad Age","Marketing, SEO & media industry","Advertising publication","United States","Global","2026-10-22","https://adage.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://adage.com/robots.txt","https://adage.com/robots.txt","","","94bad4d6bf4add3a5c8393bc2d82ccf0190e151d81f127e5e16af385e960a7de","No archived body","https://adage.com/llms.txt","https://adage.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-9da2f5ffecb2","2026-10-22 23:50:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-c53f409ab295","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"319","thedrum.com","The Drum","Marketing, SEO & media industry","Marketing publication","United Kingdom","Global","2026-10-21","https://www.thedrum.com/robots.txt","200","ok","2840","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/thedrum.com.txt","c3ecf783377cbeadb72c49ca6fc137cd2ddad4c903189597a1083fd91855637b","c3ecf783377cbeadb72c49ca6fc137cd2ddad4c903189597a1083fd91855637b","Matches source index","https://thedrum.com/robots.txt","https://www.thedrum.com/robots.txt","","","55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0","No archived body","https://thedrum.com/llms.txt","https://www.thedrum.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/thedrum.com.txt","","c3ecf783377cbeadb72c49ca6fc137cd2ddad4c903189597a1083fd91855637b","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"320","econsultancy.com","Econsultancy","Marketing, SEO & media industry","Marketing publication","United Kingdom","Global","2026-10-21","https://econsultancy.com/robots.txt","200","ok","2890","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/econsultancy.com.txt","d76601bddeb7caa6ccb8583fd02aa205034f289c200da4511c01eb87c920a61d","d76601bddeb7caa6ccb8583fd02aa205034f289c200da4511c01eb87c920a61d","Matches source index","https://econsultancy.com/robots.txt","https://econsultancy.com/robots.txt","","","a6391837c5bc99aad732a3fed55239aa0ef4b3df08a27317c0629b7885603afd","No archived body","https://econsultancy.com/llms.txt","https://econsultancy.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/econsultancy.com.txt","","d76601bddeb7caa6ccb8583fd02aa205034f289c200da4511c01eb87c920a61d","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"321","smartinsights.com","Smart Insights","Marketing, SEO & media industry","Marketing publication","United Kingdom","Global","2026-10-21","https://www.smartinsights.com/robots.txt","200","ok","179","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/smartinsights.com.txt","f2c828019d3d104d9f84e18f1b577343bbdc6b884d93df338904fed93ccf9f65","f2c828019d3d104d9f84e18f1b577343bbdc6b884d93df338904fed93ccf9f65","Matches source index","https://smartinsights.com/robots.txt","https://www.smartinsights.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://smartinsights.com/llms.txt","https://www.smartinsights.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/smartinsights.com.txt","","f2c828019d3d104d9f84e18f1b577343bbdc6b884d93df338904fed93ccf9f65","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"322","digitalmarketer.com","DigitalMarketer","Marketing, SEO & media industry","Marketing training","United States","Global","2026-10-21","https://www.digitalmarketer.com/robots.txt","200","ok","68","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/digitalmarketer.com.txt","f846585f5e636d909811dae534241f46d43ef53f33ed5e51be9a2764d3694786","f846585f5e636d909811dae534241f46d43ef53f33ed5e51be9a2764d3694786","Matches source index","https://digitalmarketer.com/robots.txt","https://www.digitalmarketer.com/robots.txt","","","fdd771629d2de82f470c4aaac823792b43034709e796c7553c2048739eca2e69","No archived body","https://digitalmarketer.com/llms.txt","https://www.digitalmarketer.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/digitalmarketer.com.txt","","f846585f5e636d909811dae534241f46d43ef53f33ed5e51be9a2764d3694786","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"323","cxl.com","CXL","Marketing, SEO & media industry","Marketing training & optimisation","United States","Global","2026-10-21","https://cxl.com/robots.txt","200","ok","342","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/cxl.com.txt","bf8b0425388af76f10f1586ee6989bc1f2822bc5048ec40ad63ec8507004926c","bf8b0425388af76f10f1586ee6989bc1f2822bc5048ec40ad63ec8507004926c","Matches source index","https://cxl.com/robots.txt","https://cxl.com/robots.txt","","","388bb9fffaee6e890eff61ba55d88a99f7b9183e5d43d59cd0895568e2859032","No archived body","https://cxl.com/llms.txt","https://cxl.com/llms.txt/","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/cxl.com.txt","","bf8b0425388af76f10f1586ee6989bc1f2822bc5048ec40ad63ec8507004926c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"324","unbounce.com","Unbounce","Marketing, SEO & media industry","Landing page tool","Canada","Global","2026-10-21","https://unbounce.com/robots.txt","200","ok","775","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/unbounce.com.txt","bb247e6d800ff6b7c08db5fe1cdf006e01ccd33430e5d3a5c8b7c6b3bd50d0cc","bb247e6d800ff6b7c08db5fe1cdf006e01ccd33430e5d3a5c8b7c6b3bd50d0cc","Matches source index","https://unbounce.com/robots.txt","https://unbounce.com/robots.txt","","","14eeb8a56067bbd1585d3046b4a97541a0fe96dc736a591e14483dd4ef23090b","No archived body","https://unbounce.com/llms.txt","https://unbounce.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/unbounce.com.txt","","bb247e6d800ff6b7c08db5fe1cdf006e01ccd33430e5d3a5c8b7c6b3bd50d0cc","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"325","leadpages.com","Leadpages","Marketing, SEO & media industry","Landing page tool","United States","Global","2026-10-21","https://leadpages.com/robots.txt","200","ok","1428","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","73","has_strict_links","78","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/leadpages.com.txt","b349eca783b4b40a9bfcb784ad69b9f8438dfc613c7011974070012a9884bebe","b349eca783b4b40a9bfcb784ad69b9f8438dfc613c7011974070012a9884bebe","Matches source index","https://leadpages.com/robots.txt","https://leadpages.com/robots.txt","llms/leadpages.com.txt","9d5ee3803d54050b38643dcb844e56fab137504be619e98e8c6aed7b82b7ca62","9d5ee3803d54050b38643dcb844e56fab137504be619e98e8c6aed7b82b7ca62","Matches source index","https://leadpages.com/llms.txt","https://leadpages.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/leadpages.com.txt","llms/leadpages.com.txt","b349eca783b4b40a9bfcb784ad69b9f8438dfc613c7011974070012a9884bebe","9d5ee3803d54050b38643dcb844e56fab137504be619e98e8c6aed7b82b7ca62","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"326","optinmonster.com","OptinMonster","Marketing, SEO & media industry","Conversion tool","United States","Global","2026-10-21","https://optinmonster.com/robots.txt","200","ok","556","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","link_list_no_h1","1886","has_strict_links","1887","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/optinmonster.com.txt","e1b6817fcc15c591910439424089f9294a6f257aa847586d5656dbf91f189ec0","e1b6817fcc15c591910439424089f9294a6f257aa847586d5656dbf91f189ec0","Matches source index","https://optinmonster.com/robots.txt","https://optinmonster.com/robots.txt","llms/optinmonster.com.txt","8917ece4b2b2a8f4b529236aa2d0f406e480956e35ae25c798d8f244af441300","8917ece4b2b2a8f4b529236aa2d0f406e480956e35ae25c798d8f244af441300","Matches source index","https://optinmonster.com/llms.txt","https://optinmonster.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/optinmonster.com.txt","llms/optinmonster.com.txt","e1b6817fcc15c591910439424089f9294a6f257aa847586d5656dbf91f189ec0","8917ece4b2b2a8f4b529236aa2d0f406e480956e35ae25c798d8f244af441300","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"327","yoast.com","Yoast","Marketing, SEO & media industry","SEO tool","Netherlands","Global","2026-10-21","https://yoast.com/robots.txt","200","ok","927","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","65","has_strict_links","65","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/yoast.com.txt","7430fe272fd171a2aec5718cc08b6ef90fdaec64617dbb779607643ace91bc06","7430fe272fd171a2aec5718cc08b6ef90fdaec64617dbb779607643ace91bc06","Matches source index","https://yoast.com/robots.txt","https://yoast.com/robots.txt","llms/yoast.com.txt","bcdd80e8634004d78b2bbd1d193f2ba1d659f1437d8a89d508e73427794615a3","bcdd80e8634004d78b2bbd1d193f2ba1d659f1437d8a89d508e73427794615a3","Matches source index","https://yoast.com/llms.txt","https://yoast.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/yoast.com.txt","llms/yoast.com.txt","7430fe272fd171a2aec5718cc08b6ef90fdaec64617dbb779607643ace91bc06","bcdd80e8634004d78b2bbd1d193f2ba1d659f1437d8a89d508e73427794615a3","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"328","screamingfrog.co.uk","Screaming Frog","Marketing, SEO & media industry","SEO tool","United Kingdom","Global","2026-10-21","https://screamingfrog.co.uk/robots.txt","200","ok","1883","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","co.uk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/screamingfrog.co.uk.txt","4daae490a5b51502ebc20ed9a7e83564d7b808a0c7647a370d8542273e55a1e1","4daae490a5b51502ebc20ed9a7e83564d7b808a0c7647a370d8542273e55a1e1","Matches source index","https://screamingfrog.co.uk/robots.txt","https://screamingfrog.co.uk/robots.txt","","","d5173c56bc0b72ac4e809aaf0130db9b32d2751d6696a0b56fee9f45941da198","No archived body","https://screamingfrog.co.uk/llms.txt","https://www.screamingfrog.co.uk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/screamingfrog.co.uk.txt","","4daae490a5b51502ebc20ed9a7e83564d7b808a0c7647a370d8542273e55a1e1","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"329","brightedge.com","BrightEdge","Marketing, SEO & media industry","SEO platform","United States","Global","2026-10-21","https://www.brightedge.com/robots.txt","200","ok","2512","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","10","has_strict_links","10","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/brightedge.com.txt","51c4292d739ece2b88c0817e3e8b6d37652bde5a0438c3630a0ad294a9c61a8f","51c4292d739ece2b88c0817e3e8b6d37652bde5a0438c3630a0ad294a9c61a8f","Matches source index","https://brightedge.com/robots.txt","https://www.brightedge.com/robots.txt","llms/brightedge.com.txt","3b70b606c1a61f63c96d79ca6ddc62b2b1e58a97e79034d636bb997eb4502898","3b70b606c1a61f63c96d79ca6ddc62b2b1e58a97e79034d636bb997eb4502898","Matches source index","https://brightedge.com/llms.txt","https://www.brightedge.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/brightedge.com.txt","llms/brightedge.com.txt","51c4292d739ece2b88c0817e3e8b6d37652bde5a0438c3630a0ad294a9c61a8f","3b70b606c1a61f63c96d79ca6ddc62b2b1e58a97e79034d636bb997eb4502898","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"330","conductor.com","Conductor","Marketing, SEO & media industry","SEO platform","United States","Global","2026-10-21","https://www.conductor.com/robots.txt","200","ok","162","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","57","has_strict_links","58","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/conductor.com.txt","0298256d0275dcbf0d4fab3077ae336c9bd79ab3f9a036d307e19bdb7e339125","0298256d0275dcbf0d4fab3077ae336c9bd79ab3f9a036d307e19bdb7e339125","Matches source index","https://conductor.com/robots.txt","https://www.conductor.com/robots.txt","llms/conductor.com.txt","520a10ef610dbc61cf3efd540cfbe124893f8821113df68365982cf9a4447212","520a10ef610dbc61cf3efd540cfbe124893f8821113df68365982cf9a4447212","Matches source index","https://conductor.com/llms.txt","https://www.conductor.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/conductor.com.txt","llms/conductor.com.txt","0298256d0275dcbf0d4fab3077ae336c9bd79ab3f9a036d307e19bdb7e339125","520a10ef610dbc61cf3efd540cfbe124893f8821113df68365982cf9a4447212","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"331","similarweb.com","Similarweb","Marketing, SEO & media industry","Web analytics tool","Israel","Global","2026-10-21","https://www.similarweb.com:443/robots.txt","200","ok","1026","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","111","has_strict_links","111","True","robots.txt redirects to host www.similarweb.com:443","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/similarweb.com.txt","f8b547fa4a960bcc83d1ef914c9fa148ac8887cafaf3e989c7cbcd14d1226d54","f8b547fa4a960bcc83d1ef914c9fa148ac8887cafaf3e989c7cbcd14d1226d54","Matches source index","https://similarweb.com/robots.txt","https://www.similarweb.com:443/robots.txt","llms/similarweb.com.txt","c177bb1202622944900ab94c4295ac88037a805579832d948545761539b715f6","c177bb1202622944900ab94c4295ac88037a805579832d948545761539b715f6","Matches source index","https://similarweb.com/llms.txt","https://www.similarweb.com:443/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/similarweb.com.txt","llms/similarweb.com.txt","f8b547fa4a960bcc83d1ef914c9fa148ac8887cafaf3e989c7cbcd14d1226d54","c177bb1202622944900ab94c4295ac88037a805579832d948545761539b715f6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"332","hotjar.com","Hotjar","Marketing, SEO & media industry","Web analytics tool","Malta","Global","2026-10-21","https://www.hotjar.com/robots.txt","200","ok","510","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hotjar.com.txt","95f1088bd49ac64b279552705fd2d444e047bc186efa58bbfb7ff7a8c1b0ee91","95f1088bd49ac64b279552705fd2d444e047bc186efa58bbfb7ff7a8c1b0ee91","Matches source index","https://hotjar.com/robots.txt","https://www.hotjar.com/robots.txt","","","d5383f8ba0ff8015391627d5ddf0a3b5775dead6bc81c46ef936f59f9a16f037","No archived body","https://hotjar.com/llms.txt","https://www.hotjar.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hotjar.com.txt","","95f1088bd49ac64b279552705fd2d444e047bc186efa58bbfb7ff7a8c1b0ee91","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"333","crazyegg.com","Crazy Egg","Marketing, SEO & media industry","Web analytics tool","United States","Global","2026-10-21","https://www.crazyegg.com:443/robots.txt","200","ok","904","valid_txt_200","True","Allowed","Partial","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","31","has_strict_links","31","True","robots.txt redirects to host www.crazyegg.com:443","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/crazyegg.com.txt","ba13cc41caf9b899c0465b283a2b255cfebc43b0a80a4a5f2489815b41477bf9","ba13cc41caf9b899c0465b283a2b255cfebc43b0a80a4a5f2489815b41477bf9","Matches source index","https://crazyegg.com/robots.txt","https://www.crazyegg.com:443/robots.txt","llms/crazyegg.com.txt","a1c65500f35a1d2f98610e838b562bf74aeb952f42a258f6972610be70182a02","a1c65500f35a1d2f98610e838b562bf74aeb952f42a258f6972610be70182a02","Matches source index","https://crazyegg.com/llms.txt","https://www.crazyegg.com:443/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/crazyegg.com.txt","llms/crazyegg.com.txt","ba13cc41caf9b899c0465b283a2b255cfebc43b0a80a4a5f2489815b41477bf9","a1c65500f35a1d2f98610e838b562bf74aeb952f42a258f6972610be70182a02","Allowed","Partial","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"334","vwo.com","VWO","Marketing, SEO & media industry","Conversion optimisation tool","India","Global","2026-10-21","https://wingify.com/robots.txt","200","ok","1856","valid_txt_200","True","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","673","has_strict_links","674","True","robots.txt redirects to host wingify.com","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/vwo.com.txt","4c5d15c932f0d381506672739b9af6357d0f1b8c779cdeb332d7a16dcf967bdc","4c5d15c932f0d381506672739b9af6357d0f1b8c779cdeb332d7a16dcf967bdc","Matches source index","https://vwo.com/robots.txt","https://wingify.com/robots.txt","llms/vwo.com.txt","13e6020c40c20d06df083ae7c0aacd5032facf48fd48640ae744688f75221948","13e6020c40c20d06df083ae7c0aacd5032facf48fd48640ae744688f75221948","Matches source index","https://vwo.com/llms.txt","https://wingify.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/vwo.com.txt","llms/vwo.com.txt","4c5d15c932f0d381506672739b9af6357d0f1b8c779cdeb332d7a16dcf967bdc","13e6020c40c20d06df083ae7c0aacd5032facf48fd48640ae744688f75221948","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"335","gtmetrix.com","GTmetrix","Marketing, SEO & media industry","Website performance tool","Canada","Global","2026-10-21","https://gtmetrix.com/robots.txt","200","ok","908","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gtmetrix.com.txt","0950710b92335224323a6d369377d5aafcccf186b34a64c95f8fef5bccf36e34","0950710b92335224323a6d369377d5aafcccf186b34a64c95f8fef5bccf36e34","Matches source index","https://gtmetrix.com/robots.txt","https://gtmetrix.com/robots.txt","","","de36ba2312013b394807c9a1e9e04e0565f785fef01ffb8884b1adcb966d7d7b","No archived body","https://gtmetrix.com/llms.txt","https://gtmetrix.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gtmetrix.com.txt","","0950710b92335224323a6d369377d5aafcccf186b34a64c95f8fef5bccf36e34","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"336","wordstream.com","WordStream","Marketing, SEO & media industry","Advertising tool & publication","United States","Global","2026-10-21","https://www.wordstream.com/robots.txt","200","ok","531","valid_txt_200","True","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","False","False","False","False","present","h1_markdown","500","has_strict_links","501","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/wordstream.com.txt","a9eab567b0eddcc5ea1bc546706c5a6611610a394ec4a3501681af2487f986bd","a9eab567b0eddcc5ea1bc546706c5a6611610a394ec4a3501681af2487f986bd","Matches source index","https://wordstream.com/robots.txt","https://www.wordstream.com/robots.txt","llms/wordstream.com.txt","cb38d7d7d4fdfea213bae8f60ccd6bf6a4faf3785b4b41c801d67211e2b8e980","cb38d7d7d4fdfea213bae8f60ccd6bf6a4faf3785b4b41c801d67211e2b8e980","Matches source index","https://wordstream.com/llms.txt","https://www.wordstream.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/wordstream.com.txt","llms/wordstream.com.txt","a9eab567b0eddcc5ea1bc546706c5a6611610a394ec4a3501681af2487f986bd","cb38d7d7d4fdfea213bae8f60ccd6bf6a4faf3785b4b41c801d67211e2b8e980","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"337","spyfu.com","SpyFu","Marketing, SEO & media industry","SEO & PPC tool","United States","Global","2026-10-21","https://www.spyfu.com/robots.txt","200","ok","22246","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/spyfu.com.txt","0225c1aaf63811de1d529d862ac5264ed60aa7893aa8e6d4fafcc40cb82a1092","0225c1aaf63811de1d529d862ac5264ed60aa7893aa8e6d4fafcc40cb82a1092","Matches source index","https://spyfu.com/robots.txt","https://www.spyfu.com/robots.txt","","","915485abe25c9b13c31239b68ba89beb42f5b658ee45c6f5c284a5d02522c33d","No archived body","https://spyfu.com/llms.txt","https://www.spyfu.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/spyfu.com.txt","","0225c1aaf63811de1d529d862ac5264ed60aa7893aa8e6d4fafcc40cb82a1092","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"338","mangools.com","Mangools","Marketing, SEO & media industry","SEO tool","Slovakia","Global","2026-10-21","https://mangools.com/robots.txt","200","ok","483","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mangools.com.txt","ef150c795fa3b94fcfdc6580071116b301a86affb5d94c269818a4a250b6d113","ef150c795fa3b94fcfdc6580071116b301a86affb5d94c269818a4a250b6d113","Matches source index","https://mangools.com/robots.txt","https://mangools.com/robots.txt","","","ff4802dc69c7b56087166acfa7a218323cc02b5f271fd7a97784e518632d72c3","No archived body","https://mangools.com/llms.txt","https://mangools.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mangools.com.txt","","ef150c795fa3b94fcfdc6580071116b301a86affb5d94c269818a4a250b6d113","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"339","buzzsumo.com","BuzzSumo","Marketing, SEO & media industry","Content marketing tool","","Global","2026-10-21","https://buzzsumo.com/robots.txt","200","ok","113","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/buzzsumo.com.txt","0c2340ccd67da26c107687f2507de67f7db4115bbdfd15515ff0f1b8c6f63214","0c2340ccd67da26c107687f2507de67f7db4115bbdfd15515ff0f1b8c6f63214","Matches source index","https://buzzsumo.com/robots.txt","https://buzzsumo.com/robots.txt","","","19a2d86ba5d0ddcb44a0b5a6be2157bf92dc959e434f08334f5b94cc5eb93efd","No archived body","https://buzzsumo.com/llms.txt","https://buzzsumo.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/buzzsumo.com.txt","","0c2340ccd67da26c107687f2507de67f7db4115bbdfd15515ff0f1b8c6f63214","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"340","growthhackers.com","GrowthHackers","Marketing, SEO & media industry","Growth marketing community","United States","Global","2026-10-21","https://growthhackers.com/robots.txt","200","ok","167","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/growthhackers.com.txt","642563163460ea42eaf4057bea358dcab50970f701648b437d46e8e846f4ee24","642563163460ea42eaf4057bea358dcab50970f701648b437d46e8e846f4ee24","Matches source index","https://growthhackers.com/robots.txt","https://growthhackers.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://growthhackers.com/llms.txt","https://growthhackers.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/growthhackers.com.txt","","642563163460ea42eaf4057bea358dcab50970f701648b437d46e8e846f4ee24","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222344-2533-0275-0000-12523e76b5c4","2026-10-22 23:44:13 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-04.json","1"
"341","dawn.com","Dawn","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.dawn.com/robots.txt","200","ok","469","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/dawn.com.txt","589fddc09199046ef6d206d5947ac1553d402b6202dd18115ae7a7ba0450a6e9","589fddc09199046ef6d206d5947ac1553d402b6202dd18115ae7a7ba0450a6e9","Matches source index","https://dawn.com/robots.txt","https://www.dawn.com/robots.txt","","","d61a5a67037b5728f7ad458957a4ead7fe3f71050d9eabf7ba470e9dca70b1bb","No archived body","https://dawn.com/llms.txt","https://www.dawn.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/dawn.com.txt","","589fddc09199046ef6d206d5947ac1553d402b6202dd18115ae7a7ba0450a6e9","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"342","geo.tv","Geo TV","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.geo.tv/robots.txt","200","ok","166","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/geo.tv.txt","1352781ddd9fb819132a08a471611008f5469747632554df11d36fd69f4a2488","1352781ddd9fb819132a08a471611008f5469747632554df11d36fd69f4a2488","Matches source index","https://geo.tv/robots.txt","https://www.geo.tv/robots.txt","llms/geo.tv.txt","06936fd1d3bfe86622da5f13f9ca9fdbf1df1e13d2bc7e5d4d4b1afeee42f361","06936fd1d3bfe86622da5f13f9ca9fdbf1df1e13d2bc7e5d4d4b1afeee42f361","Matches source index","https://geo.tv/llms.txt","https://www.geo.tv/llms.txt","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/geo.tv.txt","llms/geo.tv.txt","1352781ddd9fb819132a08a471611008f5469747632554df11d36fd69f4a2488","06936fd1d3bfe86622da5f13f9ca9fdbf1df1e13d2bc7e5d4d4b1afeee42f361","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"343","arynews.tv","ARY News","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://arynews.tv/robots.txt","200","ok","3694","valid_txt_200","True","Blocked","Blocked","Blocked","Allowed","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Allowed","Allowed","Allowed","Partial","Blocked","Partial","Allowed","Allowed","Allowed","Partial","False","True","False","False","present","h1_markdown","29","has_strict_links","29","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/arynews.tv.txt","50147028c474882353c0644c323d74404c0453ba7e1d6fd422ceb3dc46fbf45e","50147028c474882353c0644c323d74404c0453ba7e1d6fd422ceb3dc46fbf45e","Matches source index","https://arynews.tv/robots.txt","https://arynews.tv/robots.txt","llms/arynews.tv.txt","f73c263a6f69ff2c758aa255c2f571e314657219d89955c1ed4fa95233a46c0c","f73c263a6f69ff2c758aa255c2f571e314657219d89955c1ed4fa95233a46c0c","Matches source index","https://arynews.tv/llms.txt","https://arynews.tv/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/arynews.tv.txt","llms/arynews.tv.txt","50147028c474882353c0644c323d74404c0453ba7e1d6fd422ceb3dc46fbf45e","f73c263a6f69ff2c758aa255c2f571e314657219d89955c1ed4fa95233a46c0c","Blocked","Blocked","Blocked","Allowed","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Allowed","Allowed","Allowed","Partial","Blocked","Partial","Allowed","Allowed","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"344","tribune.com.pk","The Express Tribune","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://tribune.com.pk/robots.txt","200","ok","181","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/tribune.com.pk.txt","8b9d46b89c8ba4e66cb427c5e00bdd4d730afb852be0335826201b78f7c0e878","8b9d46b89c8ba4e66cb427c5e00bdd4d730afb852be0335826201b78f7c0e878","Matches source index","https://tribune.com.pk/robots.txt","https://tribune.com.pk/robots.txt","llms/tribune.com.pk.txt","ba7c90ed34090ce33a50b93c4cb6627cecc9fa586b33184ac99b9d1f442eea6c","ba7c90ed34090ce33a50b93c4cb6627cecc9fa586b33184ac99b9d1f442eea6c","Matches source index","https://tribune.com.pk/llms.txt","https://tribune.com.pk","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/tribune.com.pk.txt","llms/tribune.com.pk.txt","8b9d46b89c8ba4e66cb427c5e00bdd4d730afb852be0335826201b78f7c0e878","ba7c90ed34090ce33a50b93c4cb6627cecc9fa586b33184ac99b9d1f442eea6c","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"345","thenews.com.pk","The News International","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.thenews.com.pk/robots.txt","200","ok","227","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/thenews.com.pk.txt","f4dc5dc207e456d5ebeea1d91f01ddc86f208ee0837623f41da8c01aca7186ce","f4dc5dc207e456d5ebeea1d91f01ddc86f208ee0837623f41da8c01aca7186ce","Matches source index","https://thenews.com.pk/robots.txt","https://www.thenews.com.pk/robots.txt","","","f98e0289d259a21b0c13bb52ec90ae3c1b13c6d611068d330c4e612da84a0a23","No archived body","https://thenews.com.pk/llms.txt","https://www.thenews.com.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/thenews.com.pk.txt","","f4dc5dc207e456d5ebeea1d91f01ddc86f208ee0837623f41da8c01aca7186ce","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"346","dunyanews.tv","Dunya News","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://dunyanews.tv/robots.txt","200","ok","97","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/dunyanews.tv.txt","547a827b75041656c00fd8375d11624d740b38307e9aa7c73dc8fbe466cbd529","547a827b75041656c00fd8375d11624d740b38307e9aa7c73dc8fbe466cbd529","Matches source index","https://dunyanews.tv/robots.txt","https://dunyanews.tv/robots.txt","llms/dunyanews.tv.txt","00a5ecb9ed775f71c13a696a3ed65d3ea1dda4520328694a3cae57d90f54e12c","00a5ecb9ed775f71c13a696a3ed65d3ea1dda4520328694a3cae57d90f54e12c","Matches source index","https://dunyanews.tv/llms.txt","https://dunyanews.tv/llms.txt","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/dunyanews.tv.txt","llms/dunyanews.tv.txt","547a827b75041656c00fd8375d11624d740b38307e9aa7c73dc8fbe466cbd529","00a5ecb9ed775f71c13a696a3ed65d3ea1dda4520328694a3cae57d90f54e12c","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"347","samaa.tv","Samaa TV","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.samaa.tv/robots.txt","200","ok","215","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/samaa.tv.txt","73cc8e0f3df75f37136c9fce89dde855a8e99e5add1e6b6e7f59b769e2330081","73cc8e0f3df75f37136c9fce89dde855a8e99e5add1e6b6e7f59b769e2330081","Matches source index","https://samaa.tv/robots.txt","https://www.samaa.tv/robots.txt","","","5be2178445e11e49bdf1df0ef34f17dd6e6ccad3c3c49f5ee1fa9863559463fa","No archived body","https://samaa.tv/llms.txt","https://www.samaa.tv/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/samaa.tv.txt","","73cc8e0f3df75f37136c9fce89dde855a8e99e5add1e6b6e7f59b769e2330081","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"348","aaj.tv","Aaj TV","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.aaj.tv/robots.txt","200","ok","319","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aaj.tv.txt","2d83b2b31eda8fd735f406d9b49adcb4a6026756e978396c54e6423fad2e421d","2d83b2b31eda8fd735f406d9b49adcb4a6026756e978396c54e6423fad2e421d","Matches source index","https://aaj.tv/robots.txt","https://www.aaj.tv/robots.txt","","","88a576e2e8648df255147a042a6bd7665008cdc0685bde62691a952943bd4816","No archived body","https://aaj.tv/llms.txt","https://www.aaj.tv/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aaj.tv.txt","","2d83b2b31eda8fd735f406d9b49adcb4a6026756e978396c54e6423fad2e421d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"349","jang.com.pk","Jang","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://jang.com.pk/robots.txt","200","ok","91","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jang.com.pk.txt","6bbee37bb04de8e560dcfee1dd7f0bcc0f24ee37ea368a93e23c26543e530e28","6bbee37bb04de8e560dcfee1dd7f0bcc0f24ee37ea368a93e23c26543e530e28","Matches source index","https://jang.com.pk/robots.txt","https://jang.com.pk/robots.txt","llms/jang.com.pk.txt","1574d3967250b012232acfcaba19bb9d44daee8dbe44ff1ce57e805046fb8527","1574d3967250b012232acfcaba19bb9d44daee8dbe44ff1ce57e805046fb8527","Matches source index","https://jang.com.pk/llms.txt","https://jang.com.pk/","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jang.com.pk.txt","llms/jang.com.pk.txt","6bbee37bb04de8e560dcfee1dd7f0bcc0f24ee37ea368a93e23c26543e530e28","1574d3967250b012232acfcaba19bb9d44daee8dbe44ff1ce57e805046fb8527","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"350","brecorder.com","Business Recorder","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.brecorder.com/robots.txt","200","ok","326","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/brecorder.com.txt","77dcac50a89bfc358438434092c0aef815c3c40a3753a49213c22912a827fca5","77dcac50a89bfc358438434092c0aef815c3c40a3753a49213c22912a827fca5","Matches source index","https://brecorder.com/robots.txt","https://www.brecorder.com/robots.txt","","","e3603edcc029bcd0aeb0c29977c8e6c6787999c9c3fb6aa8f202264f37f44047","No archived body","https://brecorder.com/llms.txt","https://www.brecorder.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/brecorder.com.txt","","77dcac50a89bfc358438434092c0aef815c3c40a3753a49213c22912a827fca5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"351","pakistantoday.com.pk","Pakistan Today","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://www.pakistantoday.com.pk/robots.txt","200","ok","129","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pakistantoday.com.pk.txt","1bffb23bba8e05b9db07c2e2dacad666d0b9223ff689f7e7cc5d07f4d511c7a4","1bffb23bba8e05b9db07c2e2dacad666d0b9223ff689f7e7cc5d07f4d511c7a4","Matches source index","https://pakistantoday.com.pk/robots.txt","https://www.pakistantoday.com.pk/robots.txt","","","a53a04a143ebe24935a6d6e66fbcb712f2e21a8150a3605b8d18e5ff95b07600","No archived body","https://pakistantoday.com.pk/llms.txt","https://www.pakistantoday.com.pk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/pakistantoday.com.pk.txt","","1bffb23bba8e05b9db07c2e2dacad666d0b9223ff689f7e7cc5d07f4d511c7a4","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"352","urdupoint.com","UrduPoint","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://urdupoint.com/robots.txt","200","ok","3385","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Allowed","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/urdupoint.com.txt","25f5c793925c1540e889c139514c3a1cbc3b0229a3e7894c3dafd826010e99d5","25f5c793925c1540e889c139514c3a1cbc3b0229a3e7894c3dafd826010e99d5","Matches source index","https://urdupoint.com/robots.txt","https://urdupoint.com/robots.txt","","","ddd8e41b20dac090bfb21d6af34ba0172781fbf162c66588be54af607f2348e3","No archived body","https://urdupoint.com/llms.txt","https://urdupoint.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/urdupoint.com.txt","","25f5c793925c1540e889c139514c3a1cbc3b0229a3e7894c3dafd826010e99d5","","Blocked","Blocked","Blocked","Blocked","Allowed","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Allowed","Allowed","Allowed","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"353","hbl.com","HBL","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://www.hbl.com/robots.txt","200","ok","22","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hbl.com.txt","44f3f8eafdf064cf69acb05f23ee4dfc9ab5b3d67d81c64246bc29f7e438789a","44f3f8eafdf064cf69acb05f23ee4dfc9ab5b3d67d81c64246bc29f7e438789a","Matches source index","https://hbl.com/robots.txt","https://www.hbl.com/robots.txt","","","bcafb7cd14418bda7f7ae75742b607e7d04e4da8ad64501d208c47b9a362cab0","No archived body","https://hbl.com/llms.txt","https://www.hbl.com/404","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hbl.com.txt","","44f3f8eafdf064cf69acb05f23ee4dfc9ab5b3d67d81c64246bc29f7e438789a","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"354","meezanbank.com","Meezan Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-22","https://www.meezanbank.com/robots.txt","200","ok","208","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/meezanbank.com.txt","158f61ddedd38c7546a2a6bebfc0c29dc0930187fff6ce31fde39ef9c691c59e","158f61ddedd38c7546a2a6bebfc0c29dc0930187fff6ce31fde39ef9c691c59e","Matches source index","https://meezanbank.com/robots.txt","https://www.meezanbank.com/robots.txt","","","750d954d31afa49e23ef59bdb140edefd2ebb984786d7ee6d4265d391818c782","No archived body","https://meezanbank.com/llms.txt","https://www.meezanbank.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/meezanbank.com.txt","","158f61ddedd38c7546a2a6bebfc0c29dc0930187fff6ce31fde39ef9c691c59e","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"355","ubl.com.pk","United Bank Limited","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-22","https://ubl.com.pk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ConnectTimeout: HTTPConnectionPool(host='www.ubl.com.pk', port=80): Max retries exceeded with url: /robots.txt (Caused by ConnectTimeoutError(<HTTPCon com.pk', port=443): Max retries exceeded with url: /com.pk', port=4 llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://ubl.com.pk/robots.txt","https://ubl.com.pk/robots.txt","","","","No archived body","https://ubl.com.pk/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222350-2533-0275-0000-4e6e8883e2b9","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","","10222352-2533-0275-0000-c3fc15126da1","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"356","mcb.com.pk","MCB Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://www.mcb.com.pk/robots.txt","200","ok","63","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","com.pk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/mcb.com.pk.txt","4dd8b1e9ece94ac28129a71b819c214cc6954b113cd757e19d6679627c8619bb","4dd8b1e9ece94ac28129a71b819c214cc6954b113cd757e19d6679627c8619bb","Matches source index","https://mcb.com.pk/robots.txt","https://www.mcb.com.pk/robots.txt","llms/mcb.com.pk.txt","82201e3097f8f07a38e4931b9cada4e1ad1196fd45722d7e7c84bfb0d2fce4b5","82201e3097f8f07a38e4931b9cada4e1ad1196fd45722d7e7c84bfb0d2fce4b5","Matches source index","https://mcb.com.pk/llms.txt","https://www.mcb.com.pk/404","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/mcb.com.pk.txt","llms/mcb.com.pk.txt","4dd8b1e9ece94ac28129a71b819c214cc6954b113cd757e19d6679627c8619bb","82201e3097f8f07a38e4931b9cada4e1ad1196fd45722d7e7c84bfb0d2fce4b5","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"357","nbp.com.pk","National Bank of Pakistan","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://nbp.com.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://nbp.com.pk/robots.txt","https://nbp.com.pk/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://nbp.com.pk/llms.txt","https://nbp.com.pk/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"358","bankalfalah.com","Bank Alfalah","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://bankalfalah.com/robots.txt","200","ok","199","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bankalfalah.com.txt","34b1307d4ba8faf07c77d47f20898a500c5cd7f8a8ae5d064142151f5c05e46f","34b1307d4ba8faf07c77d47f20898a500c5cd7f8a8ae5d064142151f5c05e46f","Matches source index","https://bankalfalah.com/robots.txt","https://bankalfalah.com/robots.txt","","","af70def0a98cbc0aed737a6ef574916ffb05068960881bfd143a9a3397c67db4","No archived body","https://bankalfalah.com/llms.txt","https://bankalfalah.com/llms.txt/","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bankalfalah.com.txt","","34b1307d4ba8faf07c77d47f20898a500c5cd7f8a8ae5d064142151f5c05e46f","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"359","abl.com","Allied Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://abl.com/robots.txt","200","ok","2808","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","False","False","False","False","present","h1_markdown","85","has_strict_links","85","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/abl.com.txt","6bc55d8cdbcd14b9c8ace3c7481dc24d7922c85075a9ef1965cfb3d551b25cce","6bc55d8cdbcd14b9c8ace3c7481dc24d7922c85075a9ef1965cfb3d551b25cce","Matches source index","https://abl.com/robots.txt","https://abl.com/robots.txt","llms/abl.com.txt","337593167f9a4d4290f98d6a3ab2d2817dda2c0015459a872a45b903e58e6b73","337593167f9a4d4290f98d6a3ab2d2817dda2c0015459a872a45b903e58e6b73","Matches source index","https://abl.com/llms.txt","https://abl.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/abl.com.txt","llms/abl.com.txt","6bc55d8cdbcd14b9c8ace3c7481dc24d7922c85075a9ef1965cfb3d551b25cce","337593167f9a4d4290f98d6a3ab2d2817dda2c0015459a872a45b903e58e6b73","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Partial","Partial","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Partial","Allowed","Partial","Allowed","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"360","askaribank.com","Askari Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://askaribank.com/robots.txt","200","ok","797","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/askaribank.com.txt","2a24b6c24200bf197595e6e7fc99fc52f54207745784248af648cbd04ee0527d","2a24b6c24200bf197595e6e7fc99fc52f54207745784248af648cbd04ee0527d","Matches source index","https://askaribank.com/robots.txt","https://askaribank.com/robots.txt","","","e81e230b4096fb184e16bd727c7032eade28c6161e90c8179be904e5ed04a66c","No archived body","https://askaribank.com/llms.txt","https://askaribank.com/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/askaribank.com.txt","","2a24b6c24200bf197595e6e7fc99fc52f54207745784248af648cbd04ee0527d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-c24b5fc77d15","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"361","habibmetro.com","Habib Metropolitan Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-21","https://habibmetro.com/robots.txt","200","ok","121","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/habibmetro.com.txt","88eb991f54c02625fbeca836d7522ad3487e0b57ca2086956378b1d98d473e7a","88eb991f54c02625fbeca836d7522ad3487e0b57ca2086956378b1d98d473e7a","Matches source index","https://habibmetro.com/robots.txt","https://habibmetro.com/robots.txt","llms/habibmetro.com.txt","be117f1d6aae33c2d7a7d8dec76bd9e79bba9d35750637a918364e6e02824ba6","be117f1d6aae33c2d7a7d8dec76bd9e79bba9d35750637a918364e6e02824ba6","Matches source index","https://habibmetro.com/llms.txt","https://www.habibmetro.com/","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/habibmetro.com.txt","llms/habibmetro.com.txt","88eb991f54c02625fbeca836d7522ad3487e0b57ca2086956378b1d98d473e7a","be117f1d6aae33c2d7a7d8dec76bd9e79bba9d35750637a918364e6e02824ba6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"362","faysalbank.com","Faysal Bank","Pakistan top sites","Bank","Pakistan","Pakistan","2026-10-22","https://faysalbank.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://faysalbank.com/robots.txt","https://faysalbank.com/robots.txt","","","e6ec54d6ac5b3306da3b9e8df9850c97c76eab563569d41b7b1b910a0c92810f","No archived body","https://faysalbank.com/llms.txt","https://www.faysalbank.com/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-abc818a014c2","2026-10-22 23:50:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-c983b36fa1bc","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"363","jazz.com.pk","Jazz","Pakistan top sites","Telecom","Pakistan","Pakistan","2026-10-21","https://jazz.com.pk/robots.txt","200","ok","2733","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jazz.com.pk.txt","1d2a86960b4cb34e767b547ce8c306d12be2f4b0c2e227a43d2cbe7f9744d063","1d2a86960b4cb34e767b547ce8c306d12be2f4b0c2e227a43d2cbe7f9744d063","Matches source index","https://jazz.com.pk/robots.txt","https://jazz.com.pk/robots.txt","","","58038785f29a318bafd1176dcd6a0b5d39fc17d2e6e161f204d3591a65c1dd45","No archived body","https://jazz.com.pk/llms.txt","https://jazz.com.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jazz.com.pk.txt","","1d2a86960b4cb34e767b547ce8c306d12be2f4b0c2e227a43d2cbe7f9744d063","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"364","telenor.com.pk","Telenor Pakistan","Pakistan top sites","Telecom","Pakistan","Pakistan","2026-10-22","https://telenor.com.pk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://telenor.com.pk/robots.txt","https://telenor.com.pk/robots.txt","","","1c3ec6735b2c4408b5fcf03f00be7f6090e2fc6256c630b290f4923d875b4fee","No archived body","https://telenor.com.pk/llms.txt","https://telenor.com.pk/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-279448412a14","2026-10-22 23:50:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-523710faf1d9","2026-10-22 23:52:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"365","zong.com.pk","Zong","Pakistan top sites","Telecom","Pakistan","Pakistan","2026-10-21","https://zong.com.pk/robots.txt","200","ok","126","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/zong.com.pk.txt","5dce10a6995cf8840b13209d98ff088e052e59de2fb4c3c9d542b6c90b78b5a4","5dce10a6995cf8840b13209d98ff088e052e59de2fb4c3c9d542b6c90b78b5a4","Matches source index","https://zong.com.pk/robots.txt","https://zong.com.pk/robots.txt","llms/zong.com.pk.txt","c96c5b6e1450880895463ab16fffee89f1b1eb724a388d637f1916bc998aa832","c96c5b6e1450880895463ab16fffee89f1b1eb724a388d637f1916bc998aa832","Matches source index","https://zong.com.pk/llms.txt","https://zong.com.pk/llms.txt","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/zong.com.pk.txt","llms/zong.com.pk.txt","5dce10a6995cf8840b13209d98ff088e052e59de2fb4c3c9d542b6c90b78b5a4","c96c5b6e1450880895463ab16fffee89f1b1eb724a388d637f1916bc998aa832","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"366","ptcl.com.pk","PTCL","Pakistan top sites","Telecom","Pakistan","Pakistan","2026-10-22","https://ptcl.com.pk/robots.txt","200","ok","654","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ptcl.com.pk.txt","369229ec51ad604edcd9ee0b43f2b7c75004b6941994e487f35a197574f341ca","369229ec51ad604edcd9ee0b43f2b7c75004b6941994e487f35a197574f341ca","Matches source index","https://ptcl.com.pk/robots.txt","https://ptcl.com.pk/robots.txt","","","dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f","No archived body","https://ptcl.com.pk/llms.txt","https://ptcl.com.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/ptcl.com.pk.txt","","369229ec51ad604edcd9ee0b43f2b7c75004b6941994e487f35a197574f341ca","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"367","ufone.com","Ufone","Pakistan top sites","Telecom","Pakistan","Pakistan","2026-10-21","https://www.ufone.com/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://ufone.com/robots.txt","https://www.ufone.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://ufone.com/llms.txt","https://www.ufone.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"368","daraz.pk","Daraz Pakistan","Pakistan top sites","E-commerce","Pakistan","Pakistan","2026-10-21","https://www.daraz.pk/robots.txt","200","ok","567","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/daraz.pk.txt","0813c56750566e2f6e17ad3b45f85e754e68afeccd1a44420c64b9a1a084021d","0813c56750566e2f6e17ad3b45f85e754e68afeccd1a44420c64b9a1a084021d","Matches source index","https://daraz.pk/robots.txt","https://www.daraz.pk/robots.txt","llms/daraz.pk.txt","23ffbe3778f9cbc615454f12939385a1111cd7c3c45cbb992380e23ca1c7c05a","23ffbe3778f9cbc615454f12939385a1111cd7c3c45cbb992380e23ca1c7c05a","Matches source index","https://daraz.pk/llms.txt","https://www.daraz.pk/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/daraz.pk.txt","llms/daraz.pk.txt","0813c56750566e2f6e17ad3b45f85e754e68afeccd1a44420c64b9a1a084021d","23ffbe3778f9cbc615454f12939385a1111cd7c3c45cbb992380e23ca1c7c05a","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"369","foodpanda.pk","foodpanda Pakistan","Pakistan top sites","Food delivery","Pakistan","Pakistan","2026-10-21","https://www.foodpanda.pk/robots.txt","200","ok","909","valid_txt_200","True","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/foodpanda.pk.txt","31ce903d2858a6ac94abe387b472fa3f4a2f0056e41d8122af3bea9a95045bbd","31ce903d2858a6ac94abe387b472fa3f4a2f0056e41d8122af3bea9a95045bbd","Matches source index","https://foodpanda.pk/robots.txt","https://www.foodpanda.pk/robots.txt","","","f8a79971fd641a912f96614ae3e2604fc9cdacb39d57969b0acadccc24793cb6","No archived body","https://www.foodpanda.pk/llms.txt","https://www.foodpanda.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/foodpanda.pk.txt","","31ce903d2858a6ac94abe387b472fa3f4a2f0056e41d8122af3bea9a95045bbd","","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"370","olx.com.pk","OLX Pakistan","Pakistan top sites","Classifieds","Pakistan","Pakistan","2026-10-21","https://www.olx.com.pk/robots.txt","200","ok","1615","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com.pk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/olx.com.pk.txt","d36e75c611fdde8c632e2f3020f18cec55f635a3d23a66cff27d4b947dc71d24","d36e75c611fdde8c632e2f3020f18cec55f635a3d23a66cff27d4b947dc71d24","Matches source index","https://olx.com.pk/robots.txt","https://www.olx.com.pk/robots.txt","","","83926e1cb1995437ab9d3ced28a37f2f8a4f1635611ae154d2bb0ab7169f0755","No archived body","https://olx.com.pk/llms.txt","https://www.olx.com.pk/notfound","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/olx.com.pk.txt","","d36e75c611fdde8c632e2f3020f18cec55f635a3d23a66cff27d4b947dc71d24","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"371","zameen.com","Zameen.com","Pakistan top sites","Property","Pakistan","Pakistan","2026-10-22","https://www.zameen.com/robots.txt","200","ok","6987","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/zameen.com.txt","6790a8ea040c1f983d8e82784be4eeeded78b52a3fc02256327a491bca3b7291","6790a8ea040c1f983d8e82784be4eeeded78b52a3fc02256327a491bca3b7291","Matches source index","https://zameen.com/robots.txt","https://www.zameen.com/robots.txt","","","e5b5c3cfc697e569f473738f8b97ee78a4c4765e65bb12f47575f6a4cda4e8b8","No archived body","https://www.zameen.com/llms.txt","https://www.zameen.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/zameen.com.txt","","6790a8ea040c1f983d8e82784be4eeeded78b52a3fc02256327a491bca3b7291","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"372","pakwheels.com","PakWheels","Pakistan top sites","Automotive","Pakistan","Pakistan","2026-10-21","https://www.pakwheels.com/robots.txt","200","ok","7120","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","64","has_strict_links","64","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/pakwheels.com.txt","12ad516c8025478126d810f6a65450468c7db78615ce0fcf458cbd40265ac29e","12ad516c8025478126d810f6a65450468c7db78615ce0fcf458cbd40265ac29e","Matches source index","https://pakwheels.com/robots.txt","https://www.pakwheels.com/robots.txt","llms/pakwheels.com.txt","b3b7c5791918c96b6638955f085571f05d220b6a31bbb0a132e6efb266f7553a","b3b7c5791918c96b6638955f085571f05d220b6a31bbb0a132e6efb266f7553a","Matches source index","https://pakwheels.com/llms.txt","https://www.pakwheels.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/pakwheels.com.txt","llms/pakwheels.com.txt","12ad516c8025478126d810f6a65450468c7db78615ce0fcf458cbd40265ac29e","b3b7c5791918c96b6638955f085571f05d220b6a31bbb0a132e6efb266f7553a","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"373","rozee.pk","ROZEE.PK","Pakistan top sites","Jobs","Pakistan","Pakistan","2026-10-21","https://www.rozee.pk/robots.txt","200","ok","466","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","18","has_strict_links","18","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/rozee.pk.txt","f1d16b976f9bac8805dd71febbf14cb00ee9551896fc8b747fd14747f8730c60","f1d16b976f9bac8805dd71febbf14cb00ee9551896fc8b747fd14747f8730c60","Matches source index","https://rozee.pk/robots.txt","https://www.rozee.pk/robots.txt","llms/rozee.pk.txt","c5a45b6328103ad1c1be766e8fe32a8a4e25faba47d898ab4711005b1eae09e4","c5a45b6328103ad1c1be766e8fe32a8a4e25faba47d898ab4711005b1eae09e4","Matches source index","https://rozee.pk/llms.txt","https://www.rozee.pk/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/rozee.pk.txt","llms/rozee.pk.txt","f1d16b976f9bac8805dd71febbf14cb00ee9551896fc8b747fd14747f8730c60","c5a45b6328103ad1c1be766e8fe32a8a4e25faba47d898ab4711005b1eae09e4","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"374","priceoye.pk","PriceOye","Pakistan top sites","E-commerce","Pakistan","Pakistan","2026-10-21","https://priceoye.pk/robots.txt","200","ok","12689","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","pk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/priceoye.pk.txt","c534400c274819bb1eff683750baa5ac2fb3fc1ad6c3ee5a0e93719061d71d5a","c534400c274819bb1eff683750baa5ac2fb3fc1ad6c3ee5a0e93719061d71d5a","Matches source index","https://priceoye.pk/robots.txt","https://priceoye.pk/robots.txt","","","23eeceae8c433c9c92a6cee5270299fc22aae01fb08b0daee0b979d3bb85b0a9","No archived body","https://priceoye.pk/llms.txt","https://priceoye.pk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/priceoye.pk.txt","","c534400c274819bb1eff683750baa5ac2fb3fc1ad6c3ee5a0e93719061d71d5a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"375","nu.edu.pk","FAST National University","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://nu.edu.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://nu.edu.pk/robots.txt","https://nu.edu.pk/robots.txt","","","b495c4d7b82e7db574508927ebeea433f29d1ff773457eb7521514d311634ccf","No archived body","https://nu.edu.pk/llms.txt","https://nu.edu.pk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"376","lums.edu.pk","LUMS","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://lums.edu.pk/robots.txt","200","ok","2027","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","edu.pk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/lums.edu.pk.txt","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","Matches source index","https://lums.edu.pk/robots.txt","https://lums.edu.pk/robots.txt","","","b76e91591a00f089cccf6764fe400f23cd1584c74a6e3144aefd42b0efd2219e","No archived body","https://lums.edu.pk/llms.txt","https://lums.edu.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/lums.edu.pk.txt","","773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"377","comsats.edu.pk","COMSATS University Islamabad","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://comsats.edu.pk/robots.txt","200","ok","91","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/comsats.edu.pk.txt","9f0c75fc57aa2dab89ea472fb48c65c00aff954e9ed40cb44602532d3ff6b4eb","9f0c75fc57aa2dab89ea472fb48c65c00aff954e9ed40cb44602532d3ff6b4eb","Matches source index","https://comsats.edu.pk/robots.txt","https://comsats.edu.pk/robots.txt","","","dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f","No archived body","https://comsats.edu.pk/llms.txt","https://comsats.edu.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/comsats.edu.pk.txt","","9f0c75fc57aa2dab89ea472fb48c65c00aff954e9ed40cb44602532d3ff6b4eb","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"378","pu.edu.pk","University of the Punjab","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://pu.edu.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://pu.edu.pk/robots.txt","https://pu.edu.pk/robots.txt","","","51e1170611f4973e651fa601c8c82f8e6a4c204d3c4b35e86ad00ce88bab0664","No archived body","https://pu.edu.pk/llms.txt","https://pu.edu.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"379","nust.edu.pk","NUST","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://nust.edu.pk/robots.txt","200","ok","115","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nust.edu.pk.txt","a357cdb11f382af0dfe3c36874baa65c6d594cde6fedefa9ee0e18ba32403eaf","a357cdb11f382af0dfe3c36874baa65c6d594cde6fedefa9ee0e18ba32403eaf","Matches source index","https://nust.edu.pk/robots.txt","https://nust.edu.pk/robots.txt","","","dd7f3641f52633c8cbe6064d5b8c05057dcc002919612e3f6df1510e90b0c1b1","No archived body","https://nust.edu.pk/llms.txt","https://nust.edu.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nust.edu.pk.txt","","a357cdb11f382af0dfe3c36874baa65c6d594cde6fedefa9ee0e18ba32403eaf","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"380","uok.edu.pk","University of Karachi","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://www.uok.edu.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://uok.edu.pk/robots.txt","https://www.uok.edu.pk/robots.txt","","","510077f7288c9616a5899255d7e0a40bc8dc9a0ae78be2a05b1226e9112efa16","No archived body","https://uok.edu.pk/llms.txt","https://www.uok.edu.pk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"381","qau.edu.pk","Quaid-i-Azam University","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://qau.edu.pk/robots.txt","200","ok","189","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","edu.pk', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/qau.edu.pk.txt","5173fde4ccd5bcc5fc5f0ae1fa25863b15e7ff523fe3ad71fd89fdebd7d5bc0b","5173fde4ccd5bcc5fc5f0ae1fa25863b15e7ff523fe3ad71fd89fdebd7d5bc0b","Matches source index","https://qau.edu.pk/robots.txt","https://qau.edu.pk/robots.txt","","","23f1f6a1b9561260481cebd1caa52bc3eb9b32cd152138ceed8c6ad3cf275d34","No archived body","https://qau.edu.pk/llms.txt","https://qau.edu.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/qau.edu.pk.txt","","5173fde4ccd5bcc5fc5f0ae1fa25863b15e7ff523fe3ad71fd89fdebd7d5bc0b","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"382","vu.edu.pk","Virtual University of Pakistan","Pakistan top sites","University","Pakistan","Pakistan","2026-10-22","https://vu.edu.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","Baseline note: robots.txt fetch failed after retry: SSLError: HTTPSConnectionPool(host='www.vu.edu.pk', port=443): Max retries exceeded with url: /robots.txt (Caused by SSLError(SSLCertVerificationError edu.pk', port=443): Max retries exceeded with url: / llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No stored body; provider HTTP response in task record","https://vu.edu.pk/robots.txt","https://vu.edu.pk/robots.txt","","","","No stored body; provider HTTP response in task record","https://vu.edu.pk/llms.txt","https://vu.edu.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","no_robots","404","10222339-2533-0275-0000-dc2abfccfec3","2026-10-22 23:39:51 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/robots-02.json","1","Crawler API follow-up verification","absent","404","10222344-2533-0275-0000-609405362465","2026-10-22 23:44:12 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-04.json","1"
"383","aiou.edu.pk","Allama Iqbal Open University","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://aiou.edu.pk/robots.txt","200","ok","1586","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aiou.edu.pk.txt","8bf105e7048939b31a020ba1d6b525d019315d900bf0de458d0b30e9859256c7","8bf105e7048939b31a020ba1d6b525d019315d900bf0de458d0b30e9859256c7","Matches source index","https://aiou.edu.pk/robots.txt","https://aiou.edu.pk/robots.txt","","","56a1011d496c2af337e0de4a2695d80097134912a40075db9676be5f144c82f3","No archived body","https://aiou.edu.pk/llms.txt","https://aiou.edu.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aiou.edu.pk.txt","","8bf105e7048939b31a020ba1d6b525d019315d900bf0de458d0b30e9859256c7","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"384","aku.edu","Aga Khan University","Pakistan top sites","University","Pakistan","Pakistan","2026-10-21","https://www.aku.edu/robots.txt","200","ok","133","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aku.edu.txt","032a4532bfb1dc131de592cd151ca6d177be590415c49e2df46932a6fcf1ab3a","032a4532bfb1dc131de592cd151ca6d177be590415c49e2df46932a6fcf1ab3a","Matches source index","https://aku.edu/robots.txt","https://www.aku.edu/robots.txt","","","3262943c98d7a85a216873b7f676e1a99a14b7c93c34c6566fafcff3c4165985","No archived body","https://aku.edu/llms.txt","https://www.aku.edu/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aku.edu.txt","","032a4532bfb1dc131de592cd151ca6d177be590415c49e2df46932a6fcf1ab3a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"385","pakistan.gov.pk","Government of Pakistan","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://pakistan.gov.pk/robots.txt","200","ok","24","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pakistan.gov.pk.txt","e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f","e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f","Matches source index","https://pakistan.gov.pk/robots.txt","https://pakistan.gov.pk/robots.txt","llms/pakistan.gov.pk.txt","9291587944e411635f1545e645538ad49cb0e780bbc0eeab0b68c4001fddd5fb","9291587944e411635f1545e645538ad49cb0e780bbc0eeab0b68c4001fddd5fb","Matches source index","https://pakistan.gov.pk/llms.txt","https://pakistan.gov.pk","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/pakistan.gov.pk.txt","llms/pakistan.gov.pk.txt","e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f","9291587944e411635f1545e645538ad49cb0e780bbc0eeab0b68c4001fddd5fb","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"386","fbr.gov.pk","Federal Board of Revenue","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-21","https://fbr.gov.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed). gov.pk', port=443): Read timed out. (read timeout=15)","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://fbr.gov.pk/robots.txt","https://fbr.gov.pk/robots.txt","","","dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f","No archived body","https://fbr.gov.pk/llms.txt","https://fbr.gov.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"387","sbp.org.pk","State Bank of Pakistan","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://sbp.org.pk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 522 org.pk', port=443): Max retries exceeded with url: /org.pk', port= llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://sbp.org.pk/robots.txt","https://sbp.org.pk/robots.txt","","","02f67752788fbccf6cf5cfbe97c0c818c5940696eb79d4f3e8445c51b5a3db40","No archived body","https://www.sbp.org.pk/llms.txt","https://www.sbp.org.pk/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222347-2533-0275-0000-13fe8927bfb6","","Invalid Field: 'url - Domain Not Found'.","crawler-api-verification-pass/alternate-profile-pilot.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-a2ee538b5ba7","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"388","nadra.gov.pk","NADRA","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://nadra.gov.pk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://nadra.gov.pk/robots.txt","https://nadra.gov.pk/robots.txt","","","a1f7d64ee40726f180294658a849fec2391a5d68b56b9af2ae75f9c18cc4c80a","No archived body","https://nadra.gov.pk/llms.txt","https://nadra.gov.pk/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-ef0196b79337","2026-10-22 23:50:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-d84592366737","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"389","hec.gov.pk","Higher Education Commission Pakistan","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://www.hec.gov.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://hec.gov.pk/robots.txt","https://www.hec.gov.pk/robots.txt","","","c9729eae1d029a2dfa6d411615bc04fa2672528b976ce90557a5c42ecd51b197","No archived body","https://hec.gov.pk/llms.txt","https://www.hec.gov.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-22","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"390","punjab.gov.pk","Government of Punjab","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://punjab.gov.pk/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: robots.txt fetch failed after retry: soft HTML wall at http://www.punjab.gov.pk/robots.txt Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://punjab.gov.pk/robots.txt","https://punjab.gov.pk/robots.txt","llms/punjab.gov.pk.txt","a1c9886af847df949e3597ec157aee62def61fb3a89d26771224e338d4111e66","a1c9886af847df949e3597ec157aee62def61fb3a89d26771224e338d4111e66","Matches source index","http://punjab.gov.pk/llms.txt","https://punjab.gov.pk","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","absent","Absent / non-manifest","","llms/punjab.gov.pk.txt","","a1c9886af847df949e3597ec157aee62def61fb3a89d26771224e338d4111e66","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222350-2533-0275-0000-7c1d256e363d","","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"391","sindh.gov.pk","Government of Sindh","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-21","https://sindh.gov.pk/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed). gov.pk', port=443): Read timed out. (read timeout=15)","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://sindh.gov.pk/robots.txt","https://sindh.gov.pk/robots.txt","","","d465172175d35d493fb1633e237700022bd849fa123164790b168b8318acb090","No archived body","https://sindh.gov.pk/llms.txt","https://sindh.gov.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"392","kp.gov.pk","Government of Khyber Pakhtunkhwa","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://kp.gov.pk/robots.txt","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/kp.gov.pk.txt","ecc3048046674a7be962f5f775763c0a56fd18d9cfb3403386765039146e9eae","","Computed from archived invalid body; source index hash absent","https://kp.gov.pk/robots.txt","https://kp.gov.pk/robots.txt","llms/kp.gov.pk.txt","ecc3048046674a7be962f5f775763c0a56fd18d9cfb3403386765039146e9eae","ecc3048046674a7be962f5f775763c0a56fd18d9cfb3403386765039146e9eae","Matches source index","https://kp.gov.pk/llms.txt","https://kp.gov.pk/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/kp.gov.pk.txt","llms/kp.gov.pk.txt","ecc3048046674a7be962f5f775763c0a56fd18d9cfb3403386765039146e9eae","ecc3048046674a7be962f5f775763c0a56fd18d9cfb3403386765039146e9eae","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222350-2533-0275-0000-bc9ab4da17a4","2026-10-22 23:50:46 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"393","balochistan.gov.pk","Government of Balochistan","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-21","https://balochistan.gov.pk/robots.txt","200","ok","119","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/balochistan.gov.pk.txt","9b07b07362e6f1656306409303c000b41523ccf64b0ac58aad676ac3213036ce","9b07b07362e6f1656306409303c000b41523ccf64b0ac58aad676ac3213036ce","Matches source index","https://balochistan.gov.pk/robots.txt","https://balochistan.gov.pk/robots.txt","","","53f56331fb1958332df62bd524057ff69301882262c9754197ba8576892e455b","No archived body","https://balochistan.gov.pk/llms.txt","https://balochistan.gov.pk/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/balochistan.gov.pk.txt","","9b07b07362e6f1656306409303c000b41523ccf64b0ac58aad676ac3213036ce","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"394","ecp.gov.pk","Election Commission of Pakistan","Pakistan top sites","Government","Pakistan","Pakistan","2026-10-22","https://ecp.gov.pk/robots.txt","200","ok","26","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ecp.gov.pk.txt","9133ceedd6bece0761cc791489df7a3fdbfaaa32d6316d98b92e6f8ff76e397a","9133ceedd6bece0761cc791489df7a3fdbfaaa32d6316d98b92e6f8ff76e397a","Matches source index","https://ecp.gov.pk/robots.txt","https://ecp.gov.pk/robots.txt","llms/ecp.gov.pk.txt","fdb03a2abc11890ab5512557bd14aeb156e373d44140d137d4947847e87868c8","fdb03a2abc11890ab5512557bd14aeb156e373d44140d137d4947847e87868c8","Matches source index","https://ecp.gov.pk/llms.txt","https://ecp.gov.pk","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/ecp.gov.pk.txt","llms/ecp.gov.pk.txt","9133ceedd6bece0761cc791489df7a3fdbfaaa32d6316d98b92e6f8ff76e397a","fdb03a2abc11890ab5512557bd14aeb156e373d44140d137d4947847e87868c8","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"395","propakistani.pk","ProPakistani","Pakistan top sites","Tech media","Pakistan","Pakistan","2026-10-21","https://propakistani.pk/robots.txt","200","ok","142","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","35","has_strict_links","35","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/propakistani.pk.txt","a012514f238d269f23986247622fe3280827df46eec866a5a9458cce54e075e7","a012514f238d269f23986247622fe3280827df46eec866a5a9458cce54e075e7","Matches source index","https://propakistani.pk/robots.txt","https://propakistani.pk/robots.txt","llms/propakistani.pk.txt","22ddaf53e07b33ce9e13fb52df4c2b224f054b5bd105e1ad0e8468ec3a680651","22ddaf53e07b33ce9e13fb52df4c2b224f054b5bd105e1ad0e8468ec3a680651","Matches source index","https://propakistani.pk/llms.txt","https://propakistani.pk/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/propakistani.pk.txt","llms/propakistani.pk.txt","a012514f238d269f23986247622fe3280827df46eec866a5a9458cce54e075e7","22ddaf53e07b33ce9e13fb52df4c2b224f054b5bd105e1ad0e8468ec3a680651","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"396","techjuice.pk","TechJuice","Pakistan top sites","Tech media","Pakistan","Pakistan","2026-10-21","https://www.techjuice.pk/robots.txt","200","ok","926","valid_txt_200","True","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","51","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/techjuice.pk.txt","b7f6a0e5b3e26e69e6993a672ca3ebed38468f151fc002c3df2bdbead67e7a2d","b7f6a0e5b3e26e69e6993a672ca3ebed38468f151fc002c3df2bdbead67e7a2d","Matches source index","https://techjuice.pk/robots.txt","https://www.techjuice.pk/robots.txt","llms/techjuice.pk.txt","c4ead4cd168799b82d57ae388d93e32e47c1893aae01a209b47ac72cfe62f060","c4ead4cd168799b82d57ae388d93e32e47c1893aae01a209b47ac72cfe62f060","Matches source index","https://techjuice.pk/llms.txt","https://www.techjuice.pk/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/techjuice.pk.txt","llms/techjuice.pk.txt","b7f6a0e5b3e26e69e6993a672ca3ebed38468f151fc002c3df2bdbead67e7a2d","c4ead4cd168799b82d57ae388d93e32e47c1893aae01a209b47ac72cfe62f060","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"397","hamariweb.com","Hamariweb","Pakistan top sites","Portal","Pakistan","Pakistan","2026-10-21","https://hamariweb.com/robots.txt","200","ok","325","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/hamariweb.com.txt","f5fa37bfaf3854753aebdf683b958107a53f7a3b9296815666d9f07b3c1600ef","f5fa37bfaf3854753aebdf683b958107a53f7a3b9296815666d9f07b3c1600ef","Matches source index","https://hamariweb.com/robots.txt","https://hamariweb.com/robots.txt","","","c5bd529cd6c867213dfbbe73c23172dc5d0e7f2e8e164e268cccaf4e76df3342","No archived body","https://hamariweb.com/llms.txt","https://hamariweb.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/hamariweb.com.txt","","f5fa37bfaf3854753aebdf683b958107a53f7a3b9296815666d9f07b3c1600ef","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"398","whatmobile.com.pk","WhatMobile","Pakistan top sites","Tech media","Pakistan","Pakistan","2026-10-21","https://www.whatmobile.com.pk/robots.txt","200","ok","458","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/whatmobile.com.pk.txt","c454e47e57b65b7a8b5b74f3960e09cb451702068f79c9adbf6438b6f2f29f1b","c454e47e57b65b7a8b5b74f3960e09cb451702068f79c9adbf6438b6f2f29f1b","Matches source index","https://whatmobile.com.pk/robots.txt","https://www.whatmobile.com.pk/robots.txt","","","3a4135cbca7481eeac40996c423f12e55e81a60e92c271711a7111fbfd7b12cb","No archived body","https://whatmobile.com.pk/llms.txt","https://www.whatmobile.com.pk/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/whatmobile.com.pk.txt","","c454e47e57b65b7a8b5b74f3960e09cb451702068f79c9adbf6438b6f2f29f1b","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"399","dailypakistan.com.pk","Daily Pakistan","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://dailypakistan.com.pk/robots.txt","200","ok","1099","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/dailypakistan.com.pk.txt","628e326a6b1ebd5d91f1f55bf73b41c984662751fbd14e35553579e5f557af87","628e326a6b1ebd5d91f1f55bf73b41c984662751fbd14e35553579e5f557af87","Matches source index","https://dailypakistan.com.pk/robots.txt","https://dailypakistan.com.pk/robots.txt","","","3535c169a3db055eb59c4064d70e39db92c423dd445c89f77957409c3a6b9952","No archived body","https://dailypakistan.com.pk/llms.txt","https://dailypakistan.com.pk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/dailypakistan.com.pk.txt","","628e326a6b1ebd5d91f1f55bf73b41c984662751fbd14e35553579e5f557af87","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"400","pakobserver.net","Pakistan Observer","Pakistan top sites","News media","Pakistan","Pakistan","2026-10-21","https://pakobserver.net/robots.txt","200","ok","730","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pakobserver.net.txt","c683e643611ae27e08189ab0c5f9641838d9980e7594a834b7fb43b252d09c86","c683e643611ae27e08189ab0c5f9641838d9980e7594a834b7fb43b252d09c86","Matches source index","https://pakobserver.net/robots.txt","https://pakobserver.net/robots.txt","","","4061722379dd8f50c4311f3cb163ac284109c12367a44bf859e7eea4d68c8981","No archived body","https://pakobserver.net/llms.txt","https://pakobserver.net/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/pakobserver.net.txt","","c683e643611ae27e08189ab0c5f9641838d9980e7594a834b7fb43b252d09c86","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"401","gulfnews.com","Gulf News","Gulf/MENA","News media","UAE","Gulf/MENA","2026-10-21","https://gulfnews.com/robots.txt","200","ok","1951","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gulfnews.com.txt","191acebeb24fbc589b94c375a8a2783e9a6ca0d39680e23219f93e240a6e8149","191acebeb24fbc589b94c375a8a2783e9a6ca0d39680e23219f93e240a6e8149","Matches source index","https://gulfnews.com/robots.txt","https://gulfnews.com/robots.txt","","","0bb60ee907e44724800f407a9c087db026b87e98feaa812dffc212932c59e744","No archived body","https://gulfnews.com/llms.txt","https://gulfnews.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gulfnews.com.txt","","191acebeb24fbc589b94c375a8a2783e9a6ca0d39680e23219f93e240a6e8149","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"402","khaleejtimes.com","Khaleej Times","Gulf/MENA","News media","UAE","Gulf/MENA","2026-10-21","https://www.khaleejtimes.com/robots.txt","200","ok","3698","valid_txt_200","True","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/khaleejtimes.com.txt","263cd4029de2b41a5ebf3b85cc0e25bbb87e367ab69ce1afc1deb6812375bf8a","263cd4029de2b41a5ebf3b85cc0e25bbb87e367ab69ce1afc1deb6812375bf8a","Matches source index","https://khaleejtimes.com/robots.txt","https://www.khaleejtimes.com/robots.txt","","","71e5a63cee77072e5169ae11ea081abce45cef2cd7a29187718451328c7f4895","No archived body","https://khaleejtimes.com/llms.txt","https://www.khaleejtimes.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/khaleejtimes.com.txt","","263cd4029de2b41a5ebf3b85cc0e25bbb87e367ab69ce1afc1deb6812375bf8a","","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"403","thenationalnews.com","The National","Gulf/MENA","News media","UAE","Gulf/MENA","2026-10-21","https://www.thenationalnews.com:443/robots.txt","200","ok","3124","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host www.thenationalnews.com:443 thenationalnews.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/thenationalnews.com.txt","dd854703eec65e87434d4ff000208939d6f0ec14f74ab6e7fbb14cd75983dd01","dd854703eec65e87434d4ff000208939d6f0ec14f74ab6e7fbb14cd75983dd01","Matches source index","https://thenationalnews.com/robots.txt","https://www.thenationalnews.com:443/robots.txt","","","4aeeccc1670886625332db9b1a319c434c85e886874a440e099827916b081033","No archived body","https://thenationalnews.com/llms.txt","https://www.thenationalnews.com:443/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/thenationalnews.com.txt","","dd854703eec65e87434d4ff000208939d6f0ec14f74ab6e7fbb14cd75983dd01","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"404","emirates.com","Emirates","Gulf/MENA","Airline","UAE","Gulf/MENA","2026-10-21","https://www.emirates.com/robots.txt","200","ok","11857","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/emirates.com.txt","4fceea8c6b31ab7d5162b2b2b7f05d4942f1d68702c592632f5670705fbbe50a","4fceea8c6b31ab7d5162b2b2b7f05d4942f1d68702c592632f5670705fbbe50a","Matches source index","https://emirates.com/robots.txt","https://www.emirates.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://emirates.com/llms.txt","https://www.emirates.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/emirates.com.txt","","4fceea8c6b31ab7d5162b2b2b7f05d4942f1d68702c592632f5670705fbbe50a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"405","etihad.com","Etihad Airways","Gulf/MENA","Airline","UAE","Gulf/MENA","2026-10-22","https://etihad.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.etihad.com', port=80): Read timed out. (read timeout=20) etihad.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://etihad.com/robots.txt","https://etihad.com/robots.txt","","","","No archived body","https://etihad.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222350-2533-0275-0000-9d5033045451","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","","10222352-2533-0275-0000-834d0c91dcd9","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"406","noon.com","Noon","Gulf/MENA","E-commerce","","Gulf/MENA","2026-10-22","https://noon.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.noon.com', port=80): Read timed out. (read timeout=20) noon.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://noon.com/robots.txt","https://noon.com/robots.txt","","","","No archived body","https://noon.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-e4ed630fb9ce","2026-10-22 23:50:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-8ca542f8be11","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"407","namshi.com","Namshi","Gulf/MENA","E-commerce","UAE","Gulf/MENA","2026-10-22","https://namshi.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ReadTimeout: HTTPConnectionPool(host='www.namshi.com', port=80): Read timed out. (read timeout=20) namshi.com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://namshi.com/robots.txt","https://namshi.com/robots.txt","","","","No archived body","https://namshi.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-3b6745180f41","2026-10-22 23:50:46 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-9f9119a7db15","2026-10-22 23:52:57 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"408","emiratesnbd.com","Emirates NBD","Gulf/MENA","Bank","UAE","Gulf/MENA","2026-10-21","https://www.emiratesnbd.com/robots.txt","200","ok","84","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","407","has_strict_links","407","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/emiratesnbd.com.txt","3df958400305215655bb060c253e66b4464ee4fb59cbf06b19afb4cadc5bd190","3df958400305215655bb060c253e66b4464ee4fb59cbf06b19afb4cadc5bd190","Matches source index","https://emiratesnbd.com/robots.txt","https://www.emiratesnbd.com/robots.txt","llms/emiratesnbd.com.txt","1e59ad09838035b7a87467368fe3a6cf575eab437895d3f1ae6a43ea8dec68cc","1e59ad09838035b7a87467368fe3a6cf575eab437895d3f1ae6a43ea8dec68cc","Matches source index","https://emiratesnbd.com/llms.txt","https://www.emiratesnbd.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/emiratesnbd.com.txt","llms/emiratesnbd.com.txt","3df958400305215655bb060c253e66b4464ee4fb59cbf06b19afb4cadc5bd190","1e59ad09838035b7a87467368fe3a6cf575eab437895d3f1ae6a43ea8dec68cc","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"409","adcb.com","Abu Dhabi Commercial Bank","Gulf/MENA","Bank","UAE","Gulf/MENA","2026-10-21","https://www.adcb.com/robots.txt","200","ok","3998","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/adcb.com.txt","5b6a26e719822391a2f41b6b6882868e4d77b692b3a8526475ea57d1cde37ddf","5b6a26e719822391a2f41b6b6882868e4d77b692b3a8526475ea57d1cde37ddf","Matches source index","https://adcb.com/robots.txt","https://www.adcb.com/robots.txt","","","90b2d35cd5e08370ed20db81197dd9da1a4dbb421f71293fd5733ea49eb7b3e1","No archived body","https://adcb.com/llms.txt","https://www.adcb.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/adcb.com.txt","","5b6a26e719822391a2f41b6b6882868e4d77b692b3a8526475ea57d1cde37ddf","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"410","bankfab.com","First Abu Dhabi Bank","Gulf/MENA","Bank","UAE","Gulf/MENA","2026-10-21","https://www.bankfab.com/robots.txt","200","ok","16578","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","1","has_strict_links","29","True","bankfab.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/bankfab.com.txt","6099a2a8bacd75beb37ccbbff69c78f3e2090cd21d2310bd5c52516f5ebc75ca","6099a2a8bacd75beb37ccbbff69c78f3e2090cd21d2310bd5c52516f5ebc75ca","Matches source index","https://bankfab.com/robots.txt","https://www.bankfab.com/robots.txt","llms/bankfab.com.txt","ad801b25c22c957ba3122642deae49120cd5e552bebc5697e6afcaa5bc236982","ad801b25c22c957ba3122642deae49120cd5e552bebc5697e6afcaa5bc236982","Matches source index","https://bankfab.com/llms.txt","https://www.bankfab.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/bankfab.com.txt","llms/bankfab.com.txt","6099a2a8bacd75beb37ccbbff69c78f3e2090cd21d2310bd5c52516f5ebc75ca","ad801b25c22c957ba3122642deae49120cd5e552bebc5697e6afcaa5bc236982","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"411","eand.com","e&","Gulf/MENA","Telecom","UAE","Gulf/MENA","2026-10-22","https://www.eand.com/en/index.html","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/eand.com.txt","583412804e67b0b440921b981b9cf7dd3454374c8def64f618dca39e593c1b5e","","Computed from archived invalid body; source index hash absent","https://eand.com/robots.txt","https://www.eand.com/en/index.html","llms/eand.com.txt","583412804e67b0b440921b981b9cf7dd3454374c8def64f618dca39e593c1b5e","583412804e67b0b440921b981b9cf7dd3454374c8def64f618dca39e593c1b5e","Matches source index","https://eand.com/llms.txt","https://www.eand.com/en/index.html","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/eand.com.txt","llms/eand.com.txt","583412804e67b0b440921b981b9cf7dd3454374c8def64f618dca39e593c1b5e","583412804e67b0b440921b981b9cf7dd3454374c8def64f618dca39e593c1b5e","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222350-2533-0275-0000-e36f753e853b","2026-10-22 23:50:50 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"412","du.ae","du","Gulf/MENA","Telecom","UAE","Gulf/MENA","2026-10-21","https://www.du.ae/robots.txt","200","ok","1694","valid_txt_200","False","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/du.ae.txt","97098d637d85784cd255ac1e3b9f9f1aaf66d47e9b3828d153d0e67ca40a2700","97098d637d85784cd255ac1e3b9f9f1aaf66d47e9b3828d153d0e67ca40a2700","Matches source index","https://du.ae/robots.txt","https://www.du.ae/robots.txt","","","1f135e2ede4d2389e481d51d046e533ac485d15dc83396818d41bb81c7e84045","No archived body","https://du.ae/llms.txt","https://www.du.ae/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/du.ae.txt","","97098d637d85784cd255ac1e3b9f9f1aaf66d47e9b3828d153d0e67ca40a2700","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"413","u.ae","UAE Government Portal","Gulf/MENA","Government","UAE","Gulf/MENA","2026-10-21","https://u.ae/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed). ae', port=443): Read timed out. (read timeout=15)","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No archived body","https://u.ae/robots.txt","https://u.ae/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://u.ae/llms.txt","https://u.ae/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","no_robots","No file recorded","Yes","404","2026-10-21","absent","Absent / non-manifest","","","","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"414","bayut.com","Bayut","Gulf/MENA","Property","UAE","Gulf/MENA","2026-10-21","https://www.bayut.com/robots.txt","200","ok","8444","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","12432","has_strict_links","12432","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/bayut.com.txt","eea9812dd7cdb3033d799e419fabbd241d93fa6d305ee1de8e9c14b465b50f24","eea9812dd7cdb3033d799e419fabbd241d93fa6d305ee1de8e9c14b465b50f24","Matches source index","https://bayut.com/robots.txt","https://www.bayut.com/robots.txt","llms/bayut.com.txt","9a1ce6497bf94411f512448a7761998ef328a2dd1793800e674c26e484f11552","9a1ce6497bf94411f512448a7761998ef328a2dd1793800e674c26e484f11552","Matches source index","https://bayut.com/llms.txt","https://www.bayut.com/llms.txt","200","text/markdown; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/bayut.com.txt","llms/bayut.com.txt","eea9812dd7cdb3033d799e419fabbd241d93fa6d305ee1de8e9c14b465b50f24","9a1ce6497bf94411f512448a7761998ef328a2dd1793800e674c26e484f11552","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"415","propertyfinder.ae","Property Finder","Gulf/MENA","Property","UAE","Gulf/MENA","2026-10-21","https://www.propertyfinder.ae/robots.txt","200","ok","2575","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/propertyfinder.ae.txt","e40c3d50cb2dfaf3e7b3a9dbea8877eb1dcd8efaf7b7f7b78012054e92f6384a","e40c3d50cb2dfaf3e7b3a9dbea8877eb1dcd8efaf7b7f7b78012054e92f6384a","Matches source index","https://propertyfinder.ae/robots.txt","https://www.propertyfinder.ae/robots.txt","","","d465172175d35d493fb1633e237700022bd849fa123164790b168b8318acb090","No archived body","https://propertyfinder.ae/llms.txt","https://www.propertyfinder.ae/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/propertyfinder.ae.txt","","e40c3d50cb2dfaf3e7b3a9dbea8877eb1dcd8efaf7b7f7b78012054e92f6384a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"416","dubizzle.com","Dubizzle","Gulf/MENA","Classifieds","UAE","Gulf/MENA","2026-10-21","https://dubizzle.com/robots.txt","200","ok","2567","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","1998","has_strict_links","1998","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/dubizzle.com.txt","2702415419724219f3289a1f1365ccc2f1532a21ee34e3b134902c3552cbba71","2702415419724219f3289a1f1365ccc2f1532a21ee34e3b134902c3552cbba71","Matches source index","https://dubizzle.com/robots.txt","https://dubizzle.com/robots.txt","llms/dubizzle.com.txt","340990663601150bc964a8a07fde28a5f885c8102f14549191d6406111244f6e","340990663601150bc964a8a07fde28a5f885c8102f14549191d6406111244f6e","Matches source index","https://www.dubizzle.com/llms.txt","https://www.dubizzle.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/dubizzle.com.txt","llms/dubizzle.com.txt","2702415419724219f3289a1f1365ccc2f1532a21ee34e3b134902c3552cbba71","340990663601150bc964a8a07fde28a5f885c8102f14549191d6406111244f6e","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"417","arabnews.com","Arab News","Gulf/MENA","News media","Saudi Arabia","Gulf/MENA","2026-10-21","https://www.arabnews.com/robots.txt","200","ok","453","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","19","has_strict_links","19","True","Baseline note: com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Detected (strict rule)","v3.2-verification","November 2026","raw/arabnews.com.txt","29ac926796e630128561a91b25348b42d1814f2e78952fa79510745bffb4e89c","29ac926796e630128561a91b25348b42d1814f2e78952fa79510745bffb4e89c","Matches source index","https://arabnews.com/robots.txt","https://www.arabnews.com/robots.txt","crawler-api-verification-pass/raw/llms/arabnews.com--10222344-2533-0275-0000-180346ebccf5.txt","3ca6b6e906564c13556f8cf98e9d246cc70e9f71145a6521c6055d3ece93c296","","API-returned UTF-8 body; byte length matched provider size","https://arabnews.com/llms.txt","https://www.arabnews.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/arabnews.com.txt","","29ac926796e630128561a91b25348b42d1814f2e78952fa79510745bffb4e89c","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","present","200","10222344-2533-0275-0000-180346ebccf5","2026-10-22 23:44:11 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-04.json","1"
"418","stc.com.sa","stc","Gulf/MENA","Telecom","Saudi Arabia","Gulf/MENA","2026-10-21","https://www.stc.com.sa/robots.txt","200","ok","256","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/stc.com.sa.txt","77c514973d01351ab1e94019b84b0698a30befdd7ace25c7bdb5a26630714bad","77c514973d01351ab1e94019b84b0698a30befdd7ace25c7bdb5a26630714bad","Matches source index","https://stc.com.sa/robots.txt","https://www.stc.com.sa/robots.txt","llms/stc.com.sa.txt","53464bdda36583832049cc804d678b4712d11da25033e27cca5e932135aea835","53464bdda36583832049cc804d678b4712d11da25033e27cca5e932135aea835","Matches source index","https://stc.com.sa/llms.txt","https://www.stc.com.sa/ar/errors-pages/404.html","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/stc.com.sa.txt","llms/stc.com.sa.txt","77c514973d01351ab1e94019b84b0698a30befdd7ace25c7bdb5a26630714bad","53464bdda36583832049cc804d678b4712d11da25033e27cca5e932135aea835","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"419","my.gov.sa","Saudi National Portal","Gulf/MENA","Government","Saudi Arabia","Gulf/MENA","2026-10-22","https://my.gov.sa/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://my.gov.sa/robots.txt","https://my.gov.sa/robots.txt","","","732ede0c0ad65f323cbd83690a77b947f6817ca3e177337f981eb55ce608af6d","No archived body","https://my.gov.sa/llms.txt","https://my.gov.sa/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","403","10222350-2533-0275-0000-8f21f3fab189","2026-10-22 23:50:45 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-0296fda25910","2026-10-22 23:52:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"420","alrajhibank.com.sa","Al Rajhi Bank","Gulf/MENA","Bank","Saudi Arabia","Gulf/MENA","2026-10-21","https://alrajhibank.com.sa/robots.txt","200","ok","124","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","34","True","com.sa', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/alrajhibank.com.sa.txt","6d31c4703cda409d483597506e2024b7b0851185f63c99d713ff31c5923043b7","6d31c4703cda409d483597506e2024b7b0851185f63c99d713ff31c5923043b7","Matches source index","https://alrajhibank.com.sa/robots.txt","https://alrajhibank.com.sa/robots.txt","llms/alrajhibank.com.sa.txt","bb5c1a0a9f1e5559428b75a9ab1fda209e0724a2a48b534e893219822bddbc58","bb5c1a0a9f1e5559428b75a9ab1fda209e0724a2a48b534e893219822bddbc58","Matches source index","https://alrajhibank.com.sa/llms.txt","https://alrajhibank.com.sa/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/alrajhibank.com.sa.txt","llms/alrajhibank.com.sa.txt","6d31c4703cda409d483597506e2024b7b0851185f63c99d713ff31c5923043b7","bb5c1a0a9f1e5559428b75a9ab1fda209e0724a2a48b534e893219822bddbc58","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"421","saudia.com","Saudia","Gulf/MENA","Airline","Saudi Arabia","Gulf/MENA","2026-10-21","https://www.saudia.com/robots.txt","200","ok","855","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/saudia.com.txt","e92fa12a8b136e9a8fb3314de4e306506985545a6566430bd3768b9d7bc4e3f6","e92fa12a8b136e9a8fb3314de4e306506985545a6566430bd3768b9d7bc4e3f6","Matches source index","https://saudia.com/robots.txt","https://www.saudia.com/robots.txt","llms/saudia.com.txt","f27373a8cab132d6c161603b4c6e0ae011e0d4c534a538fa80d749b2d239f34e","f27373a8cab132d6c161603b4c6e0ae011e0d4c534a538fa80d749b2d239f34e","Matches source index","https://saudia.com/llms.txt","https://www.saudia.com/en-PK/404","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/saudia.com.txt","llms/saudia.com.txt","e92fa12a8b136e9a8fb3314de4e306506985545a6566430bd3768b9d7bc4e3f6","f27373a8cab132d6c161603b4c6e0ae011e0d4c534a538fa80d749b2d239f34e","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"422","almarai.com","Almarai","Gulf/MENA","Food and beverage","Saudi Arabia","Gulf/MENA","2026-10-21","https://www.almarai.com/robots.txt","200","ok","119","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","35","has_strict_links","35","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/almarai.com.txt","48989ea45daade66a9474e1c2ebb622ed19aacabddc0d408879db1adcea66094","48989ea45daade66a9474e1c2ebb622ed19aacabddc0d408879db1adcea66094","Matches source index","https://almarai.com/robots.txt","https://www.almarai.com/robots.txt","llms/almarai.com.txt","80022eff827b24760207296a0b7a268d3dbe509d7685faeb8176a3e0709e8ecb","80022eff827b24760207296a0b7a268d3dbe509d7685faeb8176a3e0709e8ecb","Matches source index","https://almarai.com/llms.txt","https://www.almarai.com/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/almarai.com.txt","llms/almarai.com.txt","48989ea45daade66a9474e1c2ebb622ed19aacabddc0d408879db1adcea66094","80022eff827b24760207296a0b7a268d3dbe509d7685faeb8176a3e0709e8ecb","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"423","jarir.com","Jarir Bookstore","Gulf/MENA","Retail","Saudi Arabia","Gulf/MENA","2026-10-21","https://www.jarir.com/robots.txt","200","ok","4244","valid_txt_200","True","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/jarir.com.txt","aa07ce4d0fd9611e96a738662560916fea5ab00ba5abdc91e3247f4e80173bb9","aa07ce4d0fd9611e96a738662560916fea5ab00ba5abdc91e3247f4e80173bb9","Matches source index","https://jarir.com/robots.txt","https://www.jarir.com/robots.txt","","","1f87797ce82b4e152d759da555d7ec89bd7df4f4ab4cb7062c4f4b085cb1359a","No archived body","https://jarir.com/llms.txt","https://www.jarir.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/jarir.com.txt","","aa07ce4d0fd9611e96a738662560916fea5ab00ba5abdc91e3247f4e80173bb9","","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"424","aljazeera.com","Al Jazeera","Gulf/MENA","News media","Qatar","Gulf/MENA","2026-10-21","https://www.aljazeera.com:443/robots.txt","200","ok","1834","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","False","True","True","False","absent","","","","","","Baseline note: robots.txt redirects to host www.aljazeera.com:443 llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aljazeera.com.txt","acc9277095a2c055bd1c8bdb9dd2dc3256b884456112557b6c3e2c68eaf44540","acc9277095a2c055bd1c8bdb9dd2dc3256b884456112557b6c3e2c68eaf44540","Matches source index","https://aljazeera.com/robots.txt","https://www.aljazeera.com:443/robots.txt","","","","No stored body; provider HTTP response in task record","https://aljazeera.com/llms.txt","https://www.aljazeera.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/aljazeera.com.txt","","acc9277095a2c055bd1c8bdb9dd2dc3256b884456112557b6c3e2c68eaf44540","","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222344-2533-0275-0000-8512ee511305","2026-10-22 23:44:12 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-04.json","1"
"425","qatarairways.com","Qatar Airways","Gulf/MENA","Airline","Qatar","Gulf/MENA","2026-10-22","https://qatarairways.com/robots.txt","","error","","unresolved_error","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 com', port=443): Read timed out. (read timeout=15) llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Unresolved request","No","Undetermined","Unverified","v3.2-verification","November 2026","","","","No archived body","https://qatarairways.com/robots.txt","https://qatarairways.com/robots.txt","","","f1872d417846740b096ba3efd1c3b70dc3c5eb45fb74396cfe583a11ffb40eeb","No archived body","https://qatarairways.com/llms.txt","https://qatarairways.com/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","Undetermined","","10222350-2533-0275-0000-fea93cfa2172","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","Undetermined","","10222352-2533-0275-0000-63cd35afea0f","","Target page took too long to respond.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"426","qnb.com","QNB","Gulf/MENA","Bank","Qatar","Gulf/MENA","2026-10-21","https://qnb.com/robots.txt","200","ok","108","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/qnb.com.txt","002af7e09ebaafe7d19ec48c49e50566773336575bb9bf66f886160f2d2717b0","002af7e09ebaafe7d19ec48c49e50566773336575bb9bf66f886160f2d2717b0","Matches source index","https://qnb.com/robots.txt","https://qnb.com/robots.txt","llms/qnb.com.txt","dd9413e8c5d7409e66e675bc9a0df332c0b52d5edc54584fecb730513376c1d4","dd9413e8c5d7409e66e675bc9a0df332c0b52d5edc54584fecb730513376c1d4","Matches source index","https://www.qnb.com/llms.txt","https://www.qnb.com/sites/error404.jsp","200","text/html; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/qnb.com.txt","llms/qnb.com.txt","002af7e09ebaafe7d19ec48c49e50566773336575bb9bf66f886160f2d2717b0","dd9413e8c5d7409e66e675bc9a0df332c0b52d5edc54584fecb730513376c1d4","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"427","ooredoo.com","Ooredoo","Gulf/MENA","Telecom","Qatar","Gulf/MENA","2026-10-21","https://www.ooredoo.com/robots.txt","200","ok","23","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ooredoo.com.txt","16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95","16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95","Matches source index","https://ooredoo.com/robots.txt","https://www.ooredoo.com/robots.txt","","","18c3c31d32f621136d805627f9a0e876c98e8801306f329efb6ebd3c56986aff","No archived body","https://ooredoo.com/llms.txt","https://www.ooredoo.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ooredoo.com.txt","","16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"428","youm7.com","Youm7","Gulf/MENA","News media","Egypt","Gulf/MENA","2026-10-21","https://www.youm7.com/robots.txt","200","ok","631","valid_txt_200","True","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","12","has_strict_links","14","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/youm7.com.txt","adeb678a6282974e86d8fb31a48e9807de1824fdc4a6994b7b5523345a202772","adeb678a6282974e86d8fb31a48e9807de1824fdc4a6994b7b5523345a202772","Matches source index","https://youm7.com/robots.txt","https://www.youm7.com/robots.txt","llms/youm7.com.txt","50657fc8ad574e154741f19bd4af7b597021b0bbdaef70c0e59bf75b0854decb","50657fc8ad574e154741f19bd4af7b597021b0bbdaef70c0e59bf75b0854decb","Matches source index","https://youm7.com/llms.txt","https://www.youm7.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/youm7.com.txt","llms/youm7.com.txt","adeb678a6282974e86d8fb31a48e9807de1824fdc4a6994b7b5523345a202772","50657fc8ad574e154741f19bd4af7b597021b0bbdaef70c0e59bf75b0854decb","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"429","masrawy.com","Masrawy","Gulf/MENA","News media","Egypt","Gulf/MENA","2026-10-21","https://www.masrawy.com/robots.txt","200","ok","2336","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/masrawy.com.txt","dac86e6b27edd9c60031adf93579809e9974c9bdb0461ed5c59712bd524bf276","dac86e6b27edd9c60031adf93579809e9974c9bdb0461ed5c59712bd524bf276","Matches source index","https://masrawy.com/robots.txt","https://www.masrawy.com/robots.txt","","","384fef9ae02182bfd4b38d4e19827e7408c2734894213b6d89b939a89686347d","No archived body","https://masrawy.com/llms.txt","https://www.masrawy.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/masrawy.com.txt","","dac86e6b27edd9c60031adf93579809e9974c9bdb0461ed5c59712bd524bf276","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"430","ahram.org.eg","Al-Ahram","Gulf/MENA","News media","Egypt","Gulf/MENA","2026-10-21","https://ahram.org.eg/robots.txt","200","ok","0","empty_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/ahram.org.eg.txt","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","Matches source index","https://ahram.org.eg/robots.txt","https://ahram.org.eg/robots.txt","llms/ahram.org.eg.txt","e6237dea3793c1ce60e576d8a46c57919b62d8e6fe5f34a17d496921a86c1ed9","e6237dea3793c1ce60e576d8a46c57919b62d8e6fe5f34a17d496921a86c1ed9","Matches source index","https://ahram.org.eg/llms.txt","https://ahram.org.eg/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/ahram.org.eg.txt","llms/ahram.org.eg.txt","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","e6237dea3793c1ce60e576d8a46c57919b62d8e6fe5f34a17d496921a86c1ed9","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"431","medium.com","Medium","Content platforms & communities","Publishing platform","United States","Global","2026-10-21","https://medium.com/robots.txt","200","ok","884","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/medium.com.txt","fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74","fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74","Matches source index","https://medium.com/robots.txt","https://medium.com/robots.txt","","","2d773f7ac4c4d60905ae6f66a3bd0d8a335ea5642a8efb315ea2d055fb0d0d58","No archived body","https://medium.com/llms.txt","https://medium.com/llms.txt","403","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/medium.com.txt","","fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74","","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-0caf036f2330","2026-10-22 23:52:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"432","substack.com","Substack","Content platforms & communities","Publishing platform","United States","Global","2026-10-21","https://substack.com/robots.txt","200","ok","585","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/substack.com.txt","e468186aecca926921a597e3b593f101d82741e04874f7f723007d47adb3b1e8","e468186aecca926921a597e3b593f101d82741e04874f7f723007d47adb3b1e8","Matches source index","https://substack.com/robots.txt","https://substack.com/robots.txt","","","f09e8e193975e58cc6f84df0892db9d9e8c44985de3a702910a0bd5e14a42fff","No archived body","https://substack.com/llms.txt","https://substack.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/substack.com.txt","","e468186aecca926921a597e3b593f101d82741e04874f7f723007d47adb3b1e8","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"433","reddit.com","Reddit","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://www.reddit.com/robots.txt","200","ok","538","valid_txt_200","False","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","True","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/reddit.com.txt","41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857","41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857","Matches source index","https://reddit.com/robots.txt","https://www.reddit.com/robots.txt","llms/reddit.com.txt","dcfc0557a4b2d58dbdbb99ba91f67c4eb410cbc2ae8a17d4df2dfe79e3ab0f8f","dcfc0557a4b2d58dbdbb99ba91f67c4eb410cbc2ae8a17d4df2dfe79e3ab0f8f","Matches source index","https://reddit.com/llms.txt","https://www.reddit.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/reddit.com.txt","llms/reddit.com.txt","41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857","dcfc0557a4b2d58dbdbb99ba91f67c4eb410cbc2ae8a17d4df2dfe79e3ab0f8f","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"434","quora.com","Quora","Content platforms & communities","Q&A platform","United States","Global","2026-10-21","https://www.quora.com/robots.txt","200","ok","18962","valid_txt_200","True","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/quora.com.txt","a3755528cbd265c9d927ad5b486f6a4eda99a1c642daf60929b2dfbcceb7db59","a3755528cbd265c9d927ad5b486f6a4eda99a1c642daf60929b2dfbcceb7db59","Matches source index","https://quora.com/robots.txt","https://www.quora.com/robots.txt","","","322f559cc7c82d7979837951a8f1ae399e66dde455c3df6a4dbfb56b969afb3b","No archived body","https://quora.com/llms.txt","https://www.quora.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/quora.com.txt","","a3755528cbd265c9d927ad5b486f6a4eda99a1c642daf60929b2dfbcceb7db59","","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"435","wikipedia.org","Wikipedia","Content platforms & communities","Encyclopedia","United States","Global","2026-10-22","https://en.wikipedia.org/robots.txt","200","ok","28275","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","robots.txt redirects to host en.wikipedia.org","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wikipedia.org.txt","48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","Matches source index","https://wikipedia.org/robots.txt","https://en.wikipedia.org/robots.txt","","","76563b4eeb07e461f97c018d6f0744fdaf2b0af79ebd90410e0452733d164bdf","No archived body","https://wikipedia.org/llms.txt","https://en.wikipedia.org/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/wikipedia.org.txt","","48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"436","stackoverflow.com","Stack Overflow","Content platforms & communities","Q&A platform","United States","Global","2026-10-22","https://stackoverflow.com/robots.txt","200","ok","4854","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 418 Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/stackoverflow.com--10222338-2533-0275-0000-442bc14406d7.txt","fe2b3430825eb33ca89e9ad3a2cfa448265c81c0410a40b0682f6f5ffd66b838","","API-returned UTF-8 body; byte length matched provider size","https://stackoverflow.com/robots.txt","https://stackoverflow.com/robots.txt","","","f479c6f96341be94a3143bd05a5225095d3a8cc567d579c54d35c25045d71e90","No archived body","https://stackoverflow.com/llms.txt","https://stackoverflow.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","absent","Absent / non-manifest","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-442bc14406d7","2026-10-22 23:38:48 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Frozen v3.1.2 baseline","","","","","","",""
"437","github.com","GitHub","Content platforms & communities","Code hosting platform","United States","Global","2026-10-21","https://github.com/robots.txt","200","ok","14540","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","117","has_strict_links","120","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/github.com.txt","b622e2dd9b60e8540ffd1f4b0974c9dbee73bce606f9660c6344d09357889956","b622e2dd9b60e8540ffd1f4b0974c9dbee73bce606f9660c6344d09357889956","Matches source index","https://github.com/robots.txt","https://github.com/robots.txt","llms/github.com.txt","71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","Matches source index","https://github.com/llms.txt","https://github.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/github.com.txt","llms/github.com.txt","b622e2dd9b60e8540ffd1f4b0974c9dbee73bce606f9660c6344d09357889956","71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"438","wordpress.com","WordPress.com","Content platforms & communities","Publishing platform","United States","Global","2026-10-21","https://wordpress.com/robots.txt","200","ok","1387","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","26","has_strict_links","37","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/wordpress.com.txt","6be526b4c2b4651b00044c5124a52f4ffce67d418eda94b47940f195df28a625","6be526b4c2b4651b00044c5124a52f4ffce67d418eda94b47940f195df28a625","Matches source index","https://wordpress.com/robots.txt","https://wordpress.com/robots.txt","llms/wordpress.com.txt","aa47908f5a28b561e6ccd4976c6233a395d94ab0ded5a170f0b0709a29ceb712","aa47908f5a28b561e6ccd4976c6233a395d94ab0ded5a170f0b0709a29ceb712","Matches source index","https://wordpress.com/llms.txt","https://wordpress.com/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/wordpress.com.txt","llms/wordpress.com.txt","6be526b4c2b4651b00044c5124a52f4ffce67d418eda94b47940f195df28a625","aa47908f5a28b561e6ccd4976c6233a395d94ab0ded5a170f0b0709a29ceb712","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"439","blogger.com","Blogger","Content platforms & communities","Publishing platform","United States","Global","2026-10-21","https://www.blogger.com/robots.txt","200","ok","572","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/blogger.com.txt","2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34","2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34","Matches source index","https://blogger.com/robots.txt","https://www.blogger.com/robots.txt","","","103bd81325370dd77244d0f6e2d92e8eb4852627d6a63deb7bf05a840409210a","No archived body","https://blogger.com/llms.txt","https://www.blogger.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/blogger.com.txt","","2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"440","tumblr.com","Tumblr","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://www.tumblr.com/robots.txt","200","ok","3452","valid_txt_200","True","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","False","True","False","False","absent","","","","","","Baseline note:  llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/tumblr.com.txt","2892c51fb7e06e7629ccb662ecf47dec3b0bfa6a679b4e112f25f7565ac1fcb0","2892c51fb7e06e7629ccb662ecf47dec3b0bfa6a679b4e112f25f7565ac1fcb0","Matches source index","https://tumblr.com/robots.txt","https://www.tumblr.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://tumblr.com/llms.txt","https://www.tumblr.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/tumblr.com.txt","","2892c51fb7e06e7629ccb662ecf47dec3b0bfa6a679b4e112f25f7565ac1fcb0","","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222345-2533-0275-0000-53affc9dd2ee","2026-10-22 23:45:13 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-05.json","1"
"441","youtube.com","YouTube","Content platforms & communities","Video platform","United States","Global","2026-10-21","https://www.youtube.com/robots.txt","200","ok","792","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/youtube.com.txt","96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","Matches source index","https://youtube.com/robots.txt","https://www.youtube.com/robots.txt","","","b394bacc8e03ffcbc935ecbbc730f5e602d189c8630802b6d0fd245bf0145226","No archived body","https://youtube.com/llms.txt","https://www.youtube.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/youtube.com.txt","","96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"442","vimeo.com","Vimeo","Content platforms & communities","Video platform","United States","Global","2026-10-21","https://vimeo.com/robots.txt","200","ok","22615","valid_txt_200","True","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/vimeo.com.txt","021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10","021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10","Matches source index","https://vimeo.com/robots.txt","https://vimeo.com/robots.txt","","","b5edeef357c667b0ae427fdd694f72325293dc3e387ab91ea0415115cfbf27c2","No archived body","https://vimeo.com/llms.txt","https://vimeo.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/vimeo.com.txt","","021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10","","Partial","Partial","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"443","twitch.tv","Twitch","Content platforms & communities","Streaming platform","United States","Global","2026-10-21","https://www.twitch.tv/robots.txt","200","ok","1646","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/twitch.tv.txt","34a44188490e1ff02884c384cf6e546e0fa4b2af0c8525f280b32480620002c8","34a44188490e1ff02884c384cf6e546e0fa4b2af0c8525f280b32480620002c8","Matches source index","https://twitch.tv/robots.txt","https://www.twitch.tv/robots.txt","llms/twitch.tv.txt","f763e2342b9687d1388976fb68e2af84db7f1697bed69813d7da7c7157587d0f","f763e2342b9687d1388976fb68e2af84db7f1697bed69813d7da7c7157587d0f","Matches source index","https://twitch.tv/llms.txt","https://www.twitch.tv/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/twitch.tv.txt","llms/twitch.tv.txt","34a44188490e1ff02884c384cf6e546e0fa4b2af0c8525f280b32480620002c8","f763e2342b9687d1388976fb68e2af84db7f1697bed69813d7da7c7157587d0f","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"444","discord.com","Discord","Content platforms & communities","Community platform","United States","Global","2026-10-22","https://discord.com/robots.txt","200","ok","1774","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/discord.com.txt","fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5","fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5","Matches source index","https://discord.com/robots.txt","https://discord.com/robots.txt","","","991fd201743becce4ca60002dbec2095c0ce98a37cd199468034b1c3eacb40f3","No archived body","https://discord.com/llms.txt","https://discord.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/discord.com.txt","","fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"445","pinterest.com","Pinterest","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://www.pinterest.com/robots.txt","200","ok","43462","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/pinterest.com.txt","5b62bd020de56fcdfaa92bb8e22be35fbdc082303d5b6f7a8ab6d93501a8f708","5b62bd020de56fcdfaa92bb8e22be35fbdc082303d5b6f7a8ab6d93501a8f708","Matches source index","https://pinterest.com/robots.txt","https://www.pinterest.com/robots.txt","llms/pinterest.com.txt","bfe36b8472aae913e74e9ce224787ed95069b5eef2e24776f057d8a082d6c559","bfe36b8472aae913e74e9ce224787ed95069b5eef2e24776f057d8a082d6c559","Matches source index","https://pinterest.com/llms.txt","https://www.pinterest.com/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/pinterest.com.txt","llms/pinterest.com.txt","5b62bd020de56fcdfaa92bb8e22be35fbdc082303d5b6f7a8ab6d93501a8f708","bfe36b8472aae913e74e9ce224787ed95069b5eef2e24776f057d8a082d6c559","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"446","facebook.com","Facebook","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://facebook.com/robots.txt","200","ok","26872","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/facebook.com.txt","ae34650d8199df7bcfbcd9da82b8fa911e522d3201af364ee6cae6e9872653d2","ae34650d8199df7bcfbcd9da82b8fa911e522d3201af364ee6cae6e9872653d2","Matches source index","https://facebook.com/robots.txt","https://facebook.com/robots.txt","llms/facebook.com.txt","6b988ac1937c42d3f92dda821e7a433a609094d8fb473fb5af555a986d605f26","6b988ac1937c42d3f92dda821e7a433a609094d8fb473fb5af555a986d605f26","Matches source index","https://facebook.com/llms.txt","https://www.facebook.com/llms.txt","200","text/html; charset=""utf-8""","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/facebook.com.txt","llms/facebook.com.txt","ae34650d8199df7bcfbcd9da82b8fa911e522d3201af364ee6cae6e9872653d2","6b988ac1937c42d3f92dda821e7a433a609094d8fb473fb5af555a986d605f26","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"447","instagram.com","Instagram","Content platforms & communities","Social platform","United States","Global","2026-10-22","https://www.instagram.com/robots.txt","200","ok","6184","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/instagram.com.txt","fe84b3913976aa37e983fd44f2ace9cf62973a512ae694eb18f3cf02066602b1","fe84b3913976aa37e983fd44f2ace9cf62973a512ae694eb18f3cf02066602b1","Matches source index","https://instagram.com/robots.txt","https://www.instagram.com/robots.txt","llms/instagram.com.txt","2f080de5f390f39874b3ee87b42c6427cdc1355391fa90dea7a674e129bdda89","2f080de5f390f39874b3ee87b42c6427cdc1355391fa90dea7a674e129bdda89","Matches source index","https://instagram.com/llms.txt","https://www.instagram.com/llms.txt","200","text/html; charset=""utf-8""","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/instagram.com.txt","llms/instagram.com.txt","fe84b3913976aa37e983fd44f2ace9cf62973a512ae694eb18f3cf02066602b1","2f080de5f390f39874b3ee87b42c6427cdc1355391fa90dea7a674e129bdda89","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"448","x.com","X","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://x.com/robots.txt","200","ok","2678","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/x.com.txt","14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6","14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6","Matches source index","https://x.com/robots.txt","https://x.com/robots.txt","","","23973bbdd3d50d26221196d1cdfe08ca4f69c29e00e0267be9e8b50fccf32c99","No archived body","https://x.com/llms.txt","https://x.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/x.com.txt","","14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"449","linkedin.com","LinkedIn","Content platforms & communities","Professional network","United States","Global","2026-10-22","https://www.linkedin.com/robots.txt","200","ok","120190","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","True","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/linkedin.com.txt","1eaa886c2898f40d5515d38552dddab29b48333c6a3adb5097572ad6309d709c","1eaa886c2898f40d5515d38552dddab29b48333c6a3adb5097572ad6309d709c","Matches source index","https://linkedin.com/robots.txt","https://www.linkedin.com/robots.txt","llms/linkedin.com.txt","2f2cbbb051062e082dbde91fd4f0cda71c8660d9bc85714dcaeefa7bdf2a018f","2f2cbbb051062e082dbde91fd4f0cda71c8660d9bc85714dcaeefa7bdf2a018f","Matches source index","https://linkedin.com/llms.txt","https://linkedin.com/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/linkedin.com.txt","llms/linkedin.com.txt","1eaa886c2898f40d5515d38552dddab29b48333c6a3adb5097572ad6309d709c","2f2cbbb051062e082dbde91fd4f0cda71c8660d9bc85714dcaeefa7bdf2a018f","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"450","tiktok.com","TikTok","Content platforms & communities","Video platform","","Global","2026-10-21","https://www.tiktok.com/robots.txt","200","ok","1567","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/tiktok.com.txt","835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","Matches source index","https://tiktok.com/robots.txt","https://www.tiktok.com/robots.txt","llms/tiktok.com.txt","2dc391a7083694ca9c5dcf68b31ee9a9bc9f8708258ed2000178cbc1303555f5","2dc391a7083694ca9c5dcf68b31ee9a9bc9f8708258ed2000178cbc1303555f5","Matches source index","https://tiktok.com/llms.txt","https://www.tiktok.com/404?fromUrl=/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/tiktok.com.txt","llms/tiktok.com.txt","835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","2dc391a7083694ca9c5dcf68b31ee9a9bc9f8708258ed2000178cbc1303555f5","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"451","snapchat.com","Snapchat","Content platforms & communities","Social platform","United States","Global","2026-10-21","https://www.snapchat.com:443/robots.txt","200","ok","3997","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","True","True","False","False","absent","","","","","","robots.txt redirects to host www.snapchat.com:443","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/snapchat.com.txt","ee1ba42e1ce0f9bfcbd1a7620dc7c31c50917675fd01fae255b50ded35c18f8b","ee1ba42e1ce0f9bfcbd1a7620dc7c31c50917675fd01fae255b50ded35c18f8b","Matches source index","https://snapchat.com/robots.txt","https://www.snapchat.com:443/robots.txt","","","c4edf1d8d2bc63859ffe628bab84e507a56edaa28094c087c75a973935c8d68c","No archived body","https://snapchat.com/llms.txt","https://www.snapchat.com:443/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/snapchat.com.txt","","ee1ba42e1ce0f9bfcbd1a7620dc7c31c50917675fd01fae255b50ded35c18f8b","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"452","patreon.com","Patreon","Content platforms & communities","Creator platform","United States","Global","2026-10-21","https://patreon.com/robots.txt","200","ok","759","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/patreon.com.txt","2e16dd7f2df38af17199751642ca0a7405e1fa69010813c1b54e46b2777be8ae","2e16dd7f2df38af17199751642ca0a7405e1fa69010813c1b54e46b2777be8ae","Matches source index","https://patreon.com/robots.txt","https://patreon.com/robots.txt","","","1502bfbffd875277bf634f6d856629f3044c3449b16eb2f59f7670823b5f931e","No archived body","https://patreon.com/llms.txt","https://www.patreon.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/patreon.com.txt","","2e16dd7f2df38af17199751642ca0a7405e1fa69010813c1b54e46b2777be8ae","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"453","kickstarter.com","Kickstarter","Content platforms & communities","Crowdfunding platform","United States","Global","2026-10-22","https://www.kickstarter.com/robots.txt","200","ok","289","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/kickstarter.com--10222338-2533-0275-0000-c2b6d8c668a6.txt","9bcd0860dfcc0410b1c6914840e9a498ea4a24515a911c991019fc45d4f7e287","","API-returned UTF-8 body; byte length matched provider size","https://kickstarter.com/robots.txt","https://www.kickstarter.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://kickstarter.com/llms.txt","https://www.kickstarter.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-c2b6d8c668a6","2026-10-22 23:38:47 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Crawler API follow-up verification","absent","404","10222345-2533-0275-0000-2604350f5811","2026-10-22 23:45:13 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-05.json","1"
"454","goodreads.com","Goodreads","Content platforms & communities","Book community","United States","Global","2026-10-21","https://www.goodreads.com/robots.txt","200","ok","2647","valid_txt_200","True","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","goodreads.com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/goodreads.com.txt","e1738b2f727969388be5a47f0186c2a50203292e26f4ec0186f4393d4694cab9","e1738b2f727969388be5a47f0186c2a50203292e26f4ec0186f4393d4694cab9","Matches source index","https://goodreads.com/robots.txt","https://www.goodreads.com/robots.txt","","","56a78e2cc8158f0a26eb4292b8a6535d8e4073b0b95a0796cb51159c6c4235d9","No archived body","https://goodreads.com/llms.txt","https://www.goodreads.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/goodreads.com.txt","","e1738b2f727969388be5a47f0186c2a50203292e26f4ec0186f4393d4694cab9","","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"455","soundcloud.com","SoundCloud","Content platforms & communities","Music platform","Germany","Global","2026-10-21","https://soundcloud.com/robots.txt","200","ok","1441","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/soundcloud.com.txt","7bbee7524de61fe4a1b4e1701bd55e9dde6ad96e3e5e97e04001b128ea890998","7bbee7524de61fe4a1b4e1701bd55e9dde6ad96e3e5e97e04001b128ea890998","Matches source index","https://soundcloud.com/robots.txt","https://soundcloud.com/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://soundcloud.com/llms.txt","https://soundcloud.com/llms.txt","404","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/soundcloud.com.txt","","7bbee7524de61fe4a1b4e1701bd55e9dde6ad96e3e5e97e04001b128ea890998","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"456","bandcamp.com","Bandcamp","Content platforms & communities","Music platform","United States","Global","2026-10-21","https://bandcamp.com/robots.txt","200","ok","1053","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/bandcamp.com.txt","0adec2c36c009bfb6a0a226ee5d1b52953152e189ff14554804ebf2f441483e3","0adec2c36c009bfb6a0a226ee5d1b52953152e189ff14554804ebf2f441483e3","Matches source index","https://bandcamp.com/robots.txt","https://bandcamp.com/robots.txt","","","76ffbcdbf6207476837c9bafa6b681719aa25425eecf74d58d64b1f98cff8100","No archived body","https://bandcamp.com/llms.txt","https://bandcamp.com/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/bandcamp.com.txt","","0adec2c36c009bfb6a0a226ee5d1b52953152e189ff14554804ebf2f441483e3","","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Allowed","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"457","flickr.com","Flickr","Content platforms & communities","Photo platform","United States","Global","2026-10-21","https://flickr.com/robots.txt","200","ok","2210","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","True","True","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/flickr.com.txt","93b2b4f0def4c7f2ba6075d6197551381b20997ec9812d1de87e9a18b9b1a068","93b2b4f0def4c7f2ba6075d6197551381b20997ec9812d1de87e9a18b9b1a068","Matches source index","https://flickr.com/robots.txt","https://flickr.com/robots.txt","","","64bbf65e323121b7d584e544be47f401cdb50479c371fd8caa65462edda8f9d2","No archived body","https://flickr.com/llms.txt","https://www.flickr.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/flickr.com.txt","","93b2b4f0def4c7f2ba6075d6197551381b20997ec9812d1de87e9a18b9b1a068","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Blocked","Blocked","Partial","Partial","Blocked","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"458","deviantart.com","DeviantArt","Content platforms & communities","Art community","United States","Global","2026-10-22","https://www.deviantart.com/robots.txt","200","ok","524","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note: robots.txt fetch failed after retry: ConnectionError: HTTPConnectionPool(host='www.deviantart.com', port=80): Max retries exceeded with url: /robots.txt (Caused by NameResolutionError(""HT com', port=443): Max retries exceeded with url: /com llms.txt: HTTP no response on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/deviantart.com--10222338-2533-0275-0000-e2f3829b4235.txt","d2461132fab259ccf6da5a7b4524f724ee128b81aca603d18a4cb09256612e19","","API-returned UTF-8 body; byte length matched provider size","https://deviantart.com/robots.txt","https://www.deviantart.com/robots.txt","","","","No archived body","https://deviantart.com/llms.txt","","","","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-e2f3829b4235","2026-10-22 23:38:48 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-fe0f919fcdb5","2026-10-22 23:52:55 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"459","behance.net","Behance","Content platforms & communities","Creative platform","United States","Global","2026-10-22","https://www.behance.net/robots.txt","200","ok","1317","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/behance.net--10222338-2533-0275-0000-644df10c24da.txt","1a55abe2475b1430444111a3ffceedde40c2782e40c2fda4a42ca95e2e02ba2b","","API-returned UTF-8 body; byte length matched provider size","https://behance.net/robots.txt","https://www.behance.net/robots.txt","","","","No stored body; provider HTTP response in task record","https://behance.net/llms.txt","https://www.behance.net/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-644df10c24da","2026-10-22 23:38:47 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Crawler API follow-up verification","absent","404","10222345-2533-0275-0000-7adfc44dcd23","2026-10-22 23:45:13 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-05.json","1"
"460","dribbble.com","Dribbble","Content platforms & communities","Design community","United States","Global","2026-10-21","https://dribbble.com/robots.txt","200","ok","998","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/dribbble.com.txt","a1a6a1e8019095c68d4289e4daea63c55f62c6923c644623149f0484bdf8ca50","a1a6a1e8019095c68d4289e4daea63c55f62c6923c644623149f0484bdf8ca50","Matches source index","https://dribbble.com/robots.txt","https://dribbble.com/robots.txt","","","b289627b37f02516dc12635a828d650dea1d4131d3f77fe0b28378ae67d400a1","No archived body","https://dribbble.com/llms.txt","https://dribbble.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/dribbble.com.txt","","a1a6a1e8019095c68d4289e4daea63c55f62c6923c644623149f0484bdf8ca50","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"461","stackexchange.com","Stack Exchange","Content platforms & communities","Q&A platform","United States","Global","2026-10-22","https://stackexchange.com/robots.txt","200","ok","2022","valid_txt_200","True","Partial","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Blocked","Partial","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","False","True","True","False","absent","","","","","","Baseline note: robots.txt fetch failed after retry: HTTP 418 Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/stackexchange.com--10222338-2533-0275-0000-bab813e5d57f.txt","ff0fa9258ccca8115288e29fbb2badbbbedc541e8a268ef5f48901b6e36e6d74","","API-returned UTF-8 body; byte length matched provider size","https://stackexchange.com/robots.txt","https://stackexchange.com/robots.txt","","","e341ac24afe08b6a44e85f1110330c676d4c0bbc3fa5f670b0381b9536c16cbd","No archived body","https://stackexchange.com/llms.txt","https://stackexchange.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","absent","Absent / non-manifest","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-bab813e5d57f","2026-10-22 23:38:48 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Frozen v3.1.2 baseline","","","","","","",""
"462","archive.org","Internet Archive","Content platforms & communities","Digital library","United States","Global","2026-10-22","https://archive.org/robots.txt","200","ok","238","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/archive.org.txt","83e00f46c71e9fa872694af73fa9c15a615b9052b2f10820a3d146bc506c6161","83e00f46c71e9fa872694af73fa9c15a615b9052b2f10820a3d146bc506c6161","Matches source index","https://archive.org/robots.txt","https://archive.org/robots.txt","","","91ee43d65d9cfb290e66bfaab0ca535c4015a3408bc77b6569bdb835ab4ae44c","No archived body","https://archive.org/llms.txt","https://archive.org/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/archive.org.txt","","83e00f46c71e9fa872694af73fa9c15a615b9052b2f10820a3d146bc506c6161","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"463","fandom.com","Fandom","Content platforms & communities","Wiki platform","United States","Global","2026-10-21","https://www.fandom.com/robots.txt","200","ok","546","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","True","True","False","absent","","","","","","Baseline note:  llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/fandom.com.txt","9fbad8318c2967817b0107bac71a41f9e7d805e8e4a028393eea1e887d0e88da","9fbad8318c2967817b0107bac71a41f9e7d805e8e4a028393eea1e887d0e88da","Matches source index","https://fandom.com/robots.txt","https://www.fandom.com/robots.txt","","","","No stored body; provider HTTP response in task record","https://fandom.com/llms.txt","https://www.fandom.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","unverified","Unverified","raw/fandom.com.txt","","9fbad8318c2967817b0107bac71a41f9e7d805e8e4a028393eea1e887d0e88da","","Blocked","Blocked","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Crawler API follow-up verification","absent","404","10222345-2533-0275-0000-7b684f669d00","2026-10-22 23:45:13 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/llms-05.json","1"
"464","wikihow.com","wikiHow","Content platforms & communities","How-to platform","United States","Global","2026-10-21","https://www.wikihow.com/robots.txt","200","ok","4514","valid_txt_200","True","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Allowed","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","False","True","True","False","absent","","","","","","","Usable response","Yes","At least one blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/wikihow.com.txt","77b692277243824d7874bf354fa5d6fd777619182143e767820354db2ae3e848","77b692277243824d7874bf354fa5d6fd777619182143e767820354db2ae3e848","Matches source index","https://wikihow.com/robots.txt","https://www.wikihow.com/robots.txt","","","7e2e3c550a6952d1e5cd1df66b98063ae1e669fc1298e8b957d10f25542dd834","No archived body","https://wikihow.com/llms.txt","https://www.wikihow.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/wikihow.com.txt","","77b692277243824d7874bf354fa5d6fd777619182143e767820354db2ae3e848","","Blocked","Blocked","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Allowed","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"465","instructables.com","Instructables","Content platforms & communities","DIY community","United States","Global","2026-10-21","https://www.instructables.com/robots.txt","200","ok","242","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/instructables.com.txt","aa7a142e41c5179ced95368c5dc4672e59249e0063ea94b0ce76d08d6d275e35","aa7a142e41c5179ced95368c5dc4672e59249e0063ea94b0ce76d08d6d275e35","Matches source index","https://instructables.com/robots.txt","https://www.instructables.com/robots.txt","","","53e7c30c679874ef11610ac7e920953072966061f83329d2e693a6ecd1f4becd","No archived body","https://instructables.com/llms.txt","https://www.instructables.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/instructables.com.txt","","aa7a142e41c5179ced95368c5dc4672e59249e0063ea94b0ce76d08d6d275e35","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"466","kaggle.com","Kaggle","Content platforms & communities","Data science community","United States","Global","2026-10-22","https://www.kaggle.com/robots.txt","404","no_robots","","no_robots_404","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","No file recorded","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","","","","No stored body; provider HTTP response in task record","https://kaggle.com/robots.txt","https://www.kaggle.com/robots.txt","llms/kaggle.com.txt","4597973e00367f2596656fd14b9710825d1024c7fc1a36abb4b469f012a967aa","4597973e00367f2596656fd14b9710825d1024c7fc1a36abb4b469f012a967aa","Matches source index","https://kaggle.com/llms.txt","https://kaggle.com/llms.txt","200","text/html; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/kaggle.com.txt","llms/kaggle.com.txt","bc45f89261c6a18403972a1edac734d6e8f52def2fafec1bd4a5269250d0d5b2","4597973e00367f2596656fd14b9710825d1024c7fc1a36abb4b469f012a967aa","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","no_robots","404","10222338-2533-0275-0000-90de503b55f4","2026-10-22 23:38:48 +00:00","Provider-recorded no-file HTTP response; no crawler access inference.","crawler-api-verification-pass/robots-03.json","1","Frozen v3.1.2 baseline","","","","","","",""
"467","itch.io","itch.io","Content platforms & communities","Game platform","United States","Global","2026-10-21","https://itch.io/robots.txt","200","ok","249","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/itch.io.txt","82fad5f79355e907c73074625296b9af90299aa08851eb737f1104c85887f0dc","82fad5f79355e907c73074625296b9af90299aa08851eb737f1104c85887f0dc","Matches source index","https://itch.io/robots.txt","https://itch.io/robots.txt","","","bea726b15d39c7ffe785e0a8e501f9030e4aae5a770375aaf1052e0ba3a2f3c6","No archived body","https://www.itch.io/llms.txt","https://itch.io/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/itch.io.txt","","82fad5f79355e907c73074625296b9af90299aa08851eb737f1104c85887f0dc","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"468","letterboxd.com","Letterboxd","Content platforms & communities","Film community","New Zealand","Global","2026-10-21","https://letterboxd.com/robots.txt","200","ok","3402","valid_txt_200","True","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/letterboxd.com.txt","8aba08fbd58310283cbc708989b1e019aaaef6d2f347148a6cf17551cf6ded72","8aba08fbd58310283cbc708989b1e019aaaef6d2f347148a6cf17551cf6ded72","Matches source index","https://letterboxd.com/robots.txt","https://letterboxd.com/robots.txt","","","92cbffdbe991918493d3ba394d2f223f344e0ca6b521e0a02468e3563b7f45ac","No archived body","https://letterboxd.com/llms.txt","https://letterboxd.com/llms.txt","404","text/html;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/letterboxd.com.txt","","8aba08fbd58310283cbc708989b1e019aaaef6d2f347148a6cf17551cf6ded72","","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"469","last.fm","Last.fm","Content platforms & communities","Music community","United Kingdom","Global","2026-10-21","https://www.last.fm/robots.txt","200","ok","8727","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/last.fm.txt","45ff27781a3982bd6080bc3cb3eebefe8fae0e4b643425ee76c421ebb3efd333","45ff27781a3982bd6080bc3cb3eebefe8fae0e4b643425ee76c421ebb3efd333","Matches source index","https://last.fm/robots.txt","https://www.last.fm/robots.txt","","","7846215d7d55f421cca877b1c10ca8fceded88d239deb0397bce688ed4de3e80","No archived body","https://last.fm/llms.txt","https://www.last.fm/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/last.fm.txt","","45ff27781a3982bd6080bc3cb3eebefe8fae0e4b643425ee76c421ebb3efd333","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"470","dev.to","DEV Community","Content platforms & communities","Developer community","United States","Global","2026-10-21","https://dev.to/robots.txt","200","ok","450","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","7","has_strict_links","7","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/dev.to.txt","983a578c850154f61ee9f3bead038657f65cb9fbed891eddf8e95135989e4855","983a578c850154f61ee9f3bead038657f65cb9fbed891eddf8e95135989e4855","Matches source index","https://dev.to/robots.txt","https://dev.to/robots.txt","llms/dev.to.txt","5b02356ce18e6854b4677d16a488ed1230b2574443dd37b96002e01ffb756314","5b02356ce18e6854b4677d16a488ed1230b2574443dd37b96002e01ffb756314","Matches source index","https://dev.to/llms.txt","https://dev.to/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/dev.to.txt","llms/dev.to.txt","983a578c850154f61ee9f3bead038657f65cb9fbed891eddf8e95135989e4855","5b02356ce18e6854b4677d16a488ed1230b2574443dd37b96002e01ffb756314","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"471","openai.com","OpenAI","AI companies","AI lab","United States","Global","2026-10-21","https://openai.com/robots.txt","200","ok","98","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/openai.com.txt","c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","Matches source index","https://openai.com/robots.txt","https://openai.com/robots.txt","","","02c63c0986d5a8cc197e130aa8a38f797ae7e2fc5e66bf5b46cb491d4be0b601","No archived body","https://openai.com/llms.txt","https://openai.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/openai.com.txt","","c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"472","anthropic.com","Anthropic","AI companies","AI lab","United States","Global","2026-10-21","https://www.anthropic.com/robots.txt","200","ok","71","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/anthropic.com.txt","b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7","b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7","Matches source index","https://anthropic.com/robots.txt","https://www.anthropic.com/robots.txt","","","38bcb3415576540c315e66b97dafd5ada8230ddf0248fab5c0c100e35585668b","No archived body","https://anthropic.com/llms.txt","https://www.anthropic.com/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/anthropic.com.txt","","b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"473","perplexity.ai","Perplexity","AI companies","AI search product","United States","Global","2026-10-21","https://www.perplexity.ai/robots.txt","200","ok","448","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/perplexity.ai.txt","33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256","33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256","Matches source index","https://perplexity.ai/robots.txt","https://www.perplexity.ai/robots.txt","","","72246f08fc4dbc2a91b6e9cbc7fb166126654cbd31967ccc1f0ae60c5e9ce0fa","No archived body","https://www.perplexity.ai/llms.txt","https://www.perplexity.ai/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/perplexity.ai.txt","","33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"474","mistral.ai","Mistral AI","AI companies","AI lab","France","Global","2026-10-21","https://mistral.ai/robots.txt","200","ok","70","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","27","has_strict_links","27","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/mistral.ai.txt","e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe","e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe","Matches source index","https://mistral.ai/robots.txt","https://mistral.ai/robots.txt","llms/mistral.ai.txt","4748113d69cf70d1ff9ade3d58a1e794c1d0fd42c4f379ce7c5cd25a39d1b5c9","4748113d69cf70d1ff9ade3d58a1e794c1d0fd42c4f379ce7c5cd25a39d1b5c9","Matches source index","https://mistral.ai/llms.txt","https://mistral.ai/llms.txt","200","text/plain; charset=UTF-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/mistral.ai.txt","llms/mistral.ai.txt","e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe","4748113d69cf70d1ff9ade3d58a1e794c1d0fd42c4f379ce7c5cd25a39d1b5c9","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"475","huggingface.co","Hugging Face","AI companies","AI platform","United States","Global","2026-10-21","https://huggingface.co/robots.txt","200","ok","68","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/huggingface.co.txt","045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193","045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193","Matches source index","https://huggingface.co/robots.txt","https://huggingface.co/robots.txt","","","8bfa5de865de8643f9e9b2db195442d747d459f59049efde92132d2b4186a103","No archived body","https://huggingface.co/llms.txt","https://huggingface.co/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/huggingface.co.txt","","045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"476","cohere.com","Cohere","AI companies","AI lab","Canada","Global","2026-10-21","https://cohere.com/robots.txt","200","ok","352","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","0","no_strict_links","61","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/cohere.com.txt","35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79","35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79","Matches source index","https://cohere.com/robots.txt","https://cohere.com/robots.txt","llms/cohere.com.txt","cbe3c0cbc37093be2bdb2cf0212d9ab6ece75e8b7a184f1003876061fd8418e6","cbe3c0cbc37093be2bdb2cf0212d9ab6ece75e8b7a184f1003876061fd8418e6","Matches source index","https://cohere.com/llms.txt","https://cohere.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/cohere.com.txt","llms/cohere.com.txt","35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79","cbe3c0cbc37093be2bdb2cf0212d9ab6ece75e8b7a184f1003876061fd8418e6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"477","deepmind.com","Google DeepMind","AI companies","AI lab","United Kingdom","Global","2026-10-21","https://deepmind.google/robots.txt","200","ok","69","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","22","has_strict_links","22","True","robots.txt redirects to host deepmind.google","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/deepmind.com.txt","f43be63c1571009a5f6a1364ef3bf60b48e50db989dbe9f47cc07818796b92f8","f43be63c1571009a5f6a1364ef3bf60b48e50db989dbe9f47cc07818796b92f8","Matches source index","https://deepmind.com/robots.txt","https://deepmind.google/robots.txt","llms/deepmind.com.txt","59236322acb375c4bf5a14ab211314986dd73e0ef8a4094ceb00e518c818b479","59236322acb375c4bf5a14ab211314986dd73e0ef8a4094ceb00e518c818b479","Matches source index","https://deepmind.com/llms.txt","https://deepmind.google/llms.txt","200","text/plain","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/deepmind.com.txt","llms/deepmind.com.txt","f43be63c1571009a5f6a1364ef3bf60b48e50db989dbe9f47cc07818796b92f8","59236322acb375c4bf5a14ab211314986dd73e0ef8a4094ceb00e518c818b479","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"478","stability.ai","Stability AI","AI companies","AI lab","United Kingdom","Global","2026-10-21","https://stability.ai/robots.txt","200","ok","1503","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/stability.ai.txt","6897171a85b8ebea7b235336e165861984ededa7b3e983cf1b6323e7f7ad778d","6897171a85b8ebea7b235336e165861984ededa7b3e983cf1b6323e7f7ad778d","Matches source index","https://stability.ai/robots.txt","https://stability.ai/robots.txt","","","e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","No archived body","https://stability.ai/llms.txt","https://stability.ai/llms.txt","404","text/plain;charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/stability.ai.txt","","6897171a85b8ebea7b235336e165861984ededa7b3e983cf1b6323e7f7ad778d","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"479","midjourney.com","Midjourney","AI companies","AI image product","United States","Global","2026-10-22","https://www.midjourney.com/robots.txt","200","ok","6340","valid_txt_200","True","Blocked","Partial","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Blocked","Partial","Blocked","Partial","Partial","Partial","Partial","Blocked","Partial","Partial","Partial","Partial","Partial","False","True","False","False","absent","other","0","no_strict_links","7","True","Baseline note: robots.txt fetch failed after retry: HTTP 403 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","crawler-api-verification-pass/raw/robots/midjourney.com--10222338-2533-0275-0000-6ce360122f72.txt","7918d83c331c7b7b2a35b5fbb09a1dbd6266d977a803ff29e1a986aa17cef0f7","","API-returned UTF-8 body; byte length matched provider size","https://midjourney.com/robots.txt","https://www.midjourney.com/robots.txt","crawler-api-verification-pass/raw/llms/midjourney.com--10222345-2533-0275-0000-aa796bdfe0c4.txt","2cb6fc9fec1f1fc4c3bfdf9ac03d1ba2ea20a35ea99ea0b1baa10636d7c6c9b7","","API-returned UTF-8 body; byte length matched provider size","https://midjourney.com/llms.txt","https://www.midjourney.com/llms.txt","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","error","Unresolved request","No","","2026-10-22","unverified","Unverified","","","","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Crawler API follow-up verification","ok","200","10222338-2533-0275-0000-6ce360122f72","2026-10-22 23:38:47 +00:00","Usable stored robots text; same fixed 22-token provider-aware parser.","crawler-api-verification-pass/robots-03.json","1","Crawler API follow-up verification","absent","200","10222345-2533-0275-0000-aa796bdfe0c4","2026-10-22 23:45:13 +00:00","Stored response classified under the pinned presence rule; detection does not establish quality or consumption.","crawler-api-verification-pass/llms-05.json","2"
"480","runwayml.com","Runway","AI companies","AI video product","United States","Global","2026-10-21","https://runway.com/robots.txt","200","ok","666","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","61","has_strict_links","62","True","robots.txt redirects to host runway.com","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/runwayml.com.txt","94cb7d711b21c2afde52c7a684a731da7491ec9431bb09edb6d0cabfdcae7c9e","94cb7d711b21c2afde52c7a684a731da7491ec9431bb09edb6d0cabfdcae7c9e","Matches source index","https://runwayml.com/robots.txt","https://runway.com/robots.txt","llms/runwayml.com.txt","0bb1963c43f3b9daf8cbab55d2e271859d1d23df777796b2ee650e8fad88c8a5","0bb1963c43f3b9daf8cbab55d2e271859d1d23df777796b2ee650e8fad88c8a5","Matches source index","https://runwayml.com/llms.txt","https://runway.com/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/runwayml.com.txt","llms/runwayml.com.txt","94cb7d711b21c2afde52c7a684a731da7491ec9431bb09edb6d0cabfdcae7c9e","0bb1963c43f3b9daf8cbab55d2e271859d1d23df777796b2ee650e8fad88c8a5","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"481","character.ai","Character.AI","AI companies","AI chatbot product","United States","Global","2026-10-21","https://character.ai/robots.txt","200","ok","159","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/character.ai.txt","686f48ea0a3576a71286201597336dcad052a0cb8035f37cda078cbf6ca47f18","686f48ea0a3576a71286201597336dcad052a0cb8035f37cda078cbf6ca47f18","Matches source index","https://character.ai/robots.txt","https://character.ai/robots.txt","","","1f19733818b400415b9d7c52f6ec604a1cd1cedb25f274793e24254873713054","No archived body","https://character.ai/llms.txt","https://character.ai/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/character.ai.txt","","686f48ea0a3576a71286201597336dcad052a0cb8035f37cda078cbf6ca47f18","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"482","x.ai","xAI","AI companies","AI lab","United States","Global","2026-10-21","https://x.ai/robots.txt","200","ok","225","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","ai', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/x.ai.txt","069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be","069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be","Matches source index","https://x.ai/robots.txt","https://x.ai/robots.txt","","","dfec664aa308baf0aca7bd66937cd76e80feb1194533b02c3370a823248dd747","No archived body","https://x.ai/llms.txt","https://x.ai/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/x.ai.txt","","069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"483","aleph-alpha.com","Aleph Alpha","AI companies","AI lab","Germany","Global","2026-10-21","https://aleph-alpha.com/robots.txt","200","ok","74","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/aleph-alpha.com.txt","f043c43863871bde8abd73f2682ac40892da2eb219d57090fb6a009f3d7c4d5a","f043c43863871bde8abd73f2682ac40892da2eb219d57090fb6a009f3d7c4d5a","Matches source index","https://aleph-alpha.com/robots.txt","https://aleph-alpha.com/robots.txt","","","7f2fe0eba7b65cafc7ca5399fd144001bcfe15f947e4eb16d25e2453a33385f6","No archived body","https://aleph-alpha.com/llms.txt","https://aleph-alpha.com/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/aleph-alpha.com.txt","","f043c43863871bde8abd73f2682ac40892da2eb219d57090fb6a009f3d7c4d5a","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"484","replicate.com","Replicate","AI companies","AI platform","United States","Global","2026-10-21","https://replicate.com/robots.txt","200","ok","120","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","present","h1_markdown","1","has_strict_links","2","True","com', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/replicate.com.txt","fb728bd6c9304b5bdc6c802287c1c99494f8e8afb26c8f23104f53a3d1992969","fb728bd6c9304b5bdc6c802287c1c99494f8e8afb26c8f23104f53a3d1992969","Matches source index","https://replicate.com/robots.txt","https://replicate.com/robots.txt","llms/replicate.com.txt","1b527e8c5a0dac11c63eb402f10c4ed8df1368dd91fc61ebb66bde424cdb5712","1b527e8c5a0dac11c63eb402f10c4ed8df1368dd91fc61ebb66bde424cdb5712","Matches source index","https://replicate.com/llms.txt","https://replicate.com/llms.txt","200","text/markdown; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/replicate.com.txt","llms/replicate.com.txt","fb728bd6c9304b5bdc6c802287c1c99494f8e8afb26c8f23104f53a3d1992969","1b527e8c5a0dac11c63eb402f10c4ed8df1368dd91fc61ebb66bde424cdb5712","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"485","elevenlabs.io","ElevenLabs","AI companies","AI voice product","","Global","2026-10-21","https://elevenlabs.io/robots.txt","200","ok","3222","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","present","h1_markdown","54","has_strict_links","59","True","","Usable response","Yes","None fully blocked","Detected (strict MIME)","v3.2-verification","November 2026","raw/elevenlabs.io.txt","2b87f2e936ab499d96e1a542529aea44f2cd701b8faa6504c40083ef945b59bd","2b87f2e936ab499d96e1a542529aea44f2cd701b8faa6504c40083ef945b59bd","Matches source index","https://elevenlabs.io/robots.txt","https://elevenlabs.io/robots.txt","llms/elevenlabs.io.txt","72c239d2bed14f6e18299c8761710fe1c68ec30b5a4710a0d9fd384e8cc72dc6","72c239d2bed14f6e18299c8761710fe1c68ec30b5a4710a0d9fd384e8cc72dc6","Matches source index","https://elevenlabs.io/llms.txt","https://elevenlabs.io/llms.txt","200","text/plain; charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","present","Detected (strict MIME)","raw/elevenlabs.io.txt","llms/elevenlabs.io.txt","2b87f2e936ab499d96e1a542529aea44f2cd701b8faa6504c40083ef945b59bd","72c239d2bed14f6e18299c8761710fe1c68ec30b5a4710a0d9fd384e8cc72dc6","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"486","gov.uk","GOV.UK","Government & public sector (global)","Government portal","United Kingdom","Global","2026-10-21","https://www.gov.uk/robots.txt","200","ok","1097","valid_txt_200","True","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gov.uk.txt","459835d29527b9c00f00f6af00f45c73a909af1944e81fefa153f585e1857305","459835d29527b9c00f00f6af00f45c73a909af1944e81fefa153f585e1857305","Matches source index","https://gov.uk/robots.txt","https://www.gov.uk/robots.txt","","","3b674be23eae7835ccda6d07daf892d2caefb2f5e156483e741943898839f415","No archived body","https://gov.uk/llms.txt","https://www.gov.uk/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/gov.uk.txt","","459835d29527b9c00f00f6af00f45c73a909af1944e81fefa153f585e1857305","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"487","usa.gov","USA.gov","Government & public sector (global)","Government portal","United States","Global","2026-10-21","https://www.usa.gov/robots.txt","200","ok","1558","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/usa.gov.txt","57210da4611cc41b25fb10231cdadd60520cb5ef20ad92309cdd0df6fd12377a","57210da4611cc41b25fb10231cdadd60520cb5ef20ad92309cdd0df6fd12377a","Matches source index","https://usa.gov/robots.txt","https://www.usa.gov/robots.txt","","","3dbc6c94b32fe0437ffe965b37c49f839ac5fc64006ca94dcd5020c591063741","No archived body","https://usa.gov/llms.txt","https://www.usa.gov/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/usa.gov.txt","","57210da4611cc41b25fb10231cdadd60520cb5ef20ad92309cdd0df6fd12377a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"488","canada.ca","Government of Canada","Government & public sector (global)","Government portal","Canada","Global","2026-10-22","https://www.canada.ca/robots.txt","200","ok","2730","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","canada.ca', port=443): Read timed out. (read timeout=15)","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/canada.ca.txt","5e6c293b04d4b15808595bf89bb179003f0e74ead9ca5219ec25944a25d78826","5e6c293b04d4b15808595bf89bb179003f0e74ead9ca5219ec25944a25d78826","Matches source index","https://canada.ca/robots.txt","https://www.canada.ca/robots.txt","llms/canada.ca.txt","43b9895fd0cae2374dbd18e2b2c971c27eafbdff59d0924a6bf88109dfe8e5e5","43b9895fd0cae2374dbd18e2b2c971c27eafbdff59d0924a6bf88109dfe8e5e5","Matches source index","https://canada.ca/llms.txt","https://www.canada.ca/errors/404.html","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/canada.ca.txt","llms/canada.ca.txt","5e6c293b04d4b15808595bf89bb179003f0e74ead9ca5219ec25944a25d78826","43b9895fd0cae2374dbd18e2b2c971c27eafbdff59d0924a6bf88109dfe8e5e5","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"489","australia.gov.au","Australian Government","Government & public sector (global)","Government portal","Australia","Global","2026-10-22","https://my.gov.au/en/services?redirectFrom=australia.gov.au","200","invalid_body","","invalid_body","","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","","","","","absent","","","","","","Baseline note: HTTP 200 but body is an HTML page / contains no robots directives Follow-up provenance is recorded in the verification fields.","Invalid response","No","Undetermined","Absent / non-manifest","v3.2-verification","November 2026","raw_invalid/australia.gov.au.txt","9cbf8a7f37568c2493d4dea2ab89833f92d4a2850a0d1227fec17acaf1dd771a","","Computed from archived invalid body; source index hash absent","https://australia.gov.au/robots.txt","https://my.gov.au/en/services?redirectFrom=australia.gov.au","llms/australia.gov.au.txt","513b2dd4ac63b4935156ea79b2c83bd545e059b0661f1ea31d90d5a3aa075373","513b2dd4ac63b4935156ea79b2c83bd545e059b0661f1ea31d90d5a3aa075373","Matches source index","https://australia.gov.au/llms.txt","https://my.gov.au/en/services?redirectFrom=australia.gov.au","200","text/html;charset=utf-8","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","invalid_body","Invalid response","No","200","2026-10-22","absent","Absent / non-manifest","raw_invalid/australia.gov.au.txt","llms/australia.gov.au.txt","9cbf8a7f37568c2493d4dea2ab89833f92d4a2850a0d1227fec17acaf1dd771a","513b2dd4ac63b4935156ea79b2c83bd545e059b0661f1ea31d90d5a3aa075373","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Undetermined","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","invalid_body","200","10222350-2533-0275-0000-baf35c34d90a","2026-10-22 23:50:47 +00:00","HTTP 200 returned HTML or non-directive content; policy remains undetermined.","crawler-api-verification-pass/alternate-profile-robots-02.json","2","Frozen v3.1.2 baseline","","","","","","",""
"490","europa.eu","European Union","Government & public sector (global)","Government portal","International","Global","2026-10-21","https://europa.eu/robots.txt","200","ok","4930","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/europa.eu.txt","bede8dd1fdd4b10f8a4ba2f0087e13cce1d83a52c192323b7959fd8213aeeb31","bede8dd1fdd4b10f8a4ba2f0087e13cce1d83a52c192323b7959fd8213aeeb31","Matches source index","https://europa.eu/robots.txt","https://europa.eu/robots.txt","","","e9e9054957053eba9f6da2f3af43a8b55b445d21eb8fab825c6fc09c456daac6","No archived body","https://europa.eu/llms.txt","https://european-union.europa.eu/not-found_en","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/europa.eu.txt","","bede8dd1fdd4b10f8a4ba2f0087e13cce1d83a52c192323b7959fd8213aeeb31","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"491","who.int","World Health Organization","Government & public sector (global)","International body","International","Global","2026-10-21","https://www.who.int/robots.txt","200","ok","18983","valid_txt_200","True","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/who.int.txt","c6957df319872ec37b752ec18e78f276e8957a8116af8f0e0a7cd3eb6ade78e1","c6957df319872ec37b752ec18e78f276e8957a8116af8f0e0a7cd3eb6ade78e1","Matches source index","https://who.int/robots.txt","https://www.who.int/robots.txt","","","cfd28e75ab997b2b778d95bdba91b34084fb137c85137322134ebcb8893ec399","No archived body","https://who.int/llms.txt","https://www.who.int/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/who.int.txt","","c6957df319872ec37b752ec18e78f276e8957a8116af8f0e0a7cd3eb6ade78e1","","Allowed","Allowed","Blocked","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"492","un.org","United Nations","Government & public sector (global)","International body","International","Global","2026-10-22","https://www.un.org/robots.txt","200","ok","2226","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","No robots.txt (HTTP 404); all bots recorded Allowed per RFC 9309 (no restrictions assumed).","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/un.org.txt","f7186c2f743522cf71ee3def5babc565f04af28e65ff2d6793d4ac0a294ca875","f7186c2f743522cf71ee3def5babc565f04af28e65ff2d6793d4ac0a294ca875","Matches source index","https://un.org/robots.txt","https://www.un.org/robots.txt","","","d131e38fd1342d11a54fc205c84a0ee1ce2d3d5fafa07dd6b4447f01dc4c9c77","No archived body","https://un.org/llms.txt","https://www.un.org/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/un.org.txt","","f7186c2f743522cf71ee3def5babc565f04af28e65ff2d6793d4ac0a294ca875","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"493","nasa.gov","NASA","Government & public sector (global)","Government agency","United States","Global","2026-10-21","https://www.nasa.gov/robots.txt","200","ok","69","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/nasa.gov.txt","44d72c5ad1a94bb45f25caaba917c39b39b383af3a121b59a5a7e46fbe3ccb05","44d72c5ad1a94bb45f25caaba917c39b39b383af3a121b59a5a7e46fbe3ccb05","Matches source index","https://nasa.gov/robots.txt","https://www.nasa.gov/robots.txt","","","bcd1b163c558d2d395038ece5487e28d64c43a9f5825b681e59ed104c0473591","No archived body","https://nasa.gov/llms.txt","https://www.nasa.gov/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/nasa.gov.txt","","44d72c5ad1a94bb45f25caaba917c39b39b383af3a121b59a5a7e46fbe3ccb05","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"494","data.gov","Data.gov","Government & public sector (global)","Government portal","United States","Global","2026-10-21","https://data.gov/robots.txt","200","ok","61","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/data.gov.txt","378eb3f2f929180eec5990d4b0ee8fd62bae3d02500a2056ffdb9625f67ad7b0","378eb3f2f929180eec5990d4b0ee8fd62bae3d02500a2056ffdb9625f67ad7b0","Matches source index","https://data.gov/robots.txt","https://data.gov/robots.txt","","","22281b7da135689b8aca8ccf286f70060f6ae0fe6614ebfeaf0dc1bf10c38f86","No archived body","https://data.gov/llms.txt","https://data.gov/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/data.gov.txt","","378eb3f2f929180eec5990d4b0ee8fd62bae3d02500a2056ffdb9625f67ad7b0","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"495","whitehouse.gov","The White House","Government & public sector (global)","Government portal","United States","Global","2026-10-21","https://www.whitehouse.gov/robots.txt","200","ok","111","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/whitehouse.gov.txt","44c691a48cb56ed97912daf32352c7de679f4fb2ade725a5cdc91efb7b43e496","44c691a48cb56ed97912daf32352c7de679f4fb2ade725a5cdc91efb7b43e496","Matches source index","https://whitehouse.gov/robots.txt","https://www.whitehouse.gov/robots.txt","","","17794a63f266f715fcc89af374c8259dfe26588de3455b70abd9ee955bf16b48","No archived body","https://whitehouse.gov/llms.txt","https://www.whitehouse.gov/llms.txt","404","text/html; charset=UTF-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/whitehouse.gov.txt","","44c691a48cb56ed97912daf32352c7de679f4fb2ade725a5cdc91efb7b43e496","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"496","service-public.fr","Service-Public.fr","Government & public sector (global)","Government portal","France","Global","2026-10-22","https://www.service-public.fr/robots.txt","200","ok","446","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/service-public.fr.txt","8e6ad6602d1948ed1ef50bad971ca09a656d40cd6d3709aa485c5c4e674da322","8e6ad6602d1948ed1ef50bad971ca09a656d40cd6d3709aa485c5c4e674da322","Matches source index","https://service-public.fr/robots.txt","https://www.service-public.fr/robots.txt","","","c0a936cf2e9e60afb57562dc4da086c70caa29e1391b2bdfa385c80ea3f68640","No archived body","https://service-public.fr/llms.txt","https://www.service-public.gouv.fr/llms.txt","404","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/service-public.fr.txt","","8e6ad6602d1948ed1ef50bad971ca09a656d40cd6d3709aa485c5c4e674da322","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"497","india.gov.in","National Portal of India","Government & public sector (global)","Government portal","India","Global","2026-10-21","https://www.india.gov.in/robots.txt","200","ok","82","valid_txt_200","False","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/india.gov.in.txt","ffa629920770a5a777d9e9bb6239a5ba8734882e4f722a039e6390ca55b916e9","ffa629920770a5a777d9e9bb6239a5ba8734882e4f722a039e6390ca55b916e9","Matches source index","https://india.gov.in/robots.txt","https://www.india.gov.in/robots.txt","","","e4d5d1d41f3454fc0571e8243cf8942cee0bc54f2f36df43a0e3636b49ca7010","No archived body","https://india.gov.in/llms.txt","https://www.india.gov.in/llms.txt","404","text/html; charset=utf-8","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-21","absent","Absent / non-manifest","raw/india.gov.in.txt","","ffa629920770a5a777d9e9bb6239a5ba8734882e4f722a039e6390ca55b916e9","","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Allowed","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"498","gov.br","Gov.br","Government & public sector (global)","Government portal","Brazil","Global","2026-10-22","https://www.gov.br/robots.txt","200","ok","2102","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gov.br.txt","665417a8e7d2184fe1effefd6a242111682a9e59e94c4aa8af17e9c5487a090a","665417a8e7d2184fe1effefd6a242111682a9e59e94c4aa8af17e9c5487a090a","Matches source index","https://gov.br/robots.txt","https://www.gov.br/robots.txt","","","6bbcea2b37bd092cc60604157a0ceec8944866fc2478f7af8aea3c3df40bc258","No archived body","https://gov.br/llms.txt","https://www.gov.br/llms.txt","404","application/json","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/gov.br.txt","","665417a8e7d2184fe1effefd6a242111682a9e59e94c4aa8af17e9c5487a090a","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""
"499","gov.za","South African Government","Government & public sector (global)","Government portal","South Africa","Global","2026-10-22","https://www.gov.za/robots.txt","200","ok","1706","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","unverified","","","","","","Baseline note: za', port=443): Max retries exceeded with url: /za', port=443) at 0 llms.txt: HTTP 403 on all variants. Follow-up provenance is recorded in the verification fields.","Usable response","Yes","None fully blocked","Unverified","v3.2-verification","November 2026","raw/gov.za.txt","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","Matches source index","https://gov.za/robots.txt","https://www.gov.za/robots.txt","","","25fb23868ebf48348f9e438e00cb9b9d9b3a054f32482a781c762cc4f9cc6393","No archived body","https://www.gov.za/llms.txt","https://www.gov.za/llms.txt","403","text/html","","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","unverified","Unverified","raw/gov.za.txt","","2a787f20489a257692bad5f16ae5ceb28876cd38d8d9cde9c84a2d3835e30bab","","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","Undetermined","403","10222352-2533-0275-0000-6edaa8d9a0c0","2026-10-22 23:52:56 +00:00","No usable target response body/status, HTTP denial, rate limit or server error; remains undetermined.","crawler-api-verification-pass/alternate-profile-llms-03.json","2"
"500","gov.sg","Singapore Government","Government & public sector (global)","Government portal","Singapore","Global","2026-10-22","https://www.gov.sg/robots.txt","200","ok","82","valid_txt_200","False","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","False","False","False","False","absent","","","","","","","Usable response","Yes","None fully blocked","Absent / non-manifest","v3.2-verification","November 2026","raw/gov.sg.txt","dbdd6709fb1f9948f11752e2195da0f9974c0161bc93d50f113d054b4f7d8d2a","dbdd6709fb1f9948f11752e2195da0f9974c0161bc93d50f113d054b4f7d8d2a","Matches source index","https://gov.sg/robots.txt","https://www.gov.sg/robots.txt","llms/gov.sg.txt","b09497dd5c1d8b8f56230aa09ec28fd03825d91399bb405c33f3f1d05d257edf","b09497dd5c1d8b8f56230aa09ec28fd03825d91399bb405c33f3f1d05d257edf","Matches source index","https://gov.sg/llms.txt","https://www.gov.sg","200","text/html","False","Archive paths refer to files inside the research package, not live website links. Hashes show stored-byte consistency, not independent timestamp verification.","ok","Usable response","Yes","200","2026-10-22","absent","Absent / non-manifest","raw/gov.sg.txt","llms/gov.sg.txt","dbdd6709fb1f9948f11752e2195da0f9974c0161bc93d50f113d054b4f7d8d2a","b09497dd5c1d8b8f56230aa09ec28fd03825d91399bb405c33f3f1d05d257edf","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Partial","Frozen baseline plus Crawler API follow-up; mixed observation times/vantages","Frozen v3.1.2 baseline","","","","","","","","Frozen v3.1.2 baseline","","","","","","",""