{"eid":"commoncrawl","name":"Common Crawl","domain":"commoncrawl.org","resolved_by":"domain","topics":["crawler-scraper","storage"],"n_records":2,"best_vorder":1,"review":null,"aliases":["Common Crawl","Common Crawl (the petabyte corpus — read side only)","commoncrawl.org"],"records":[{"rid":"crawler-scraper/commoncrawl","card":{"rid":"crawler-scraper/commoncrawl","eid":"commoncrawl","topic":"crawler-scraper","slug":"commoncrawl","provider":"Common Crawl","url":"https://commoncrawl.org/","domain":"commoncrawl.org","category":"archive","tier":null,"verdict":"qualifies-with-caveat","vorder":1,"grp":"usable","confidence":"measured","binding":"request-rate","identity_kind":"anonymous","identity_basis":"stated","needs_card":0,"needs_phone":0,"needs_id":0,"needs_institution":0,"country":null,"crypto":null,"interfaces":"rest","egress_metered":0,"per_object_bytes":null,"retention_days":null,"verified_at":"2026-10-22T00:00:00.000Z","verified_basis":"live","checked_on":"2026-10-22","stale_on":"2026-12-21","n_evidence":2,"usd_month":null,"usd_per_tb_month":null,"summary":"keyless index and WARC access","duplicate_of":null,"n_unknown":3},"record":{"topic":"crawler-scraper","slug":"commoncrawl","provider":"Common Crawl","url":"https://commoncrawl.org/","category":"archive","verdict":"qualifies-with-caveat","confidence":"measured","confidence_note":"","verified_at":"2026-10-22T00:00:00.000Z","free_tier":{"stated":"keyless index and WARC access","unit":"requests","metered_on":"request-count","source_url":"https://commoncrawl.org/","read_on":"2026-10-22","notes":""},"per_object_cap":{"stated":"not applicable: the unit is a WARC record","bytes":null,"source_url":"https://commoncrawl.org/","notes":""},"retention":{"stated":"each crawl keeps its own index; up to 12 are asked","days":null,"source_url":"https://commoncrawl.org/","notes":""},"egress":{"terms":"not applicable: the archived body, read from a WARC","metered":false,"free_allowance":null,"price_per_tb":null,"source_url":"https://commoncrawl.org/"},"interface":["rest"],"interface_detail":"Index API per crawl on index.commoncrawl.org; WARC files on data.commoncrawl.org","tooling":"not stated","chaining":"The last archive the broker asks, and the only one that can answer more true on an otherwise empty listing","limits":[{"kind":"request-rate","value":"about 1 request a second on the index, per the platform research report","source":"platform research report 2026-10-20-archive.md"},{"kind":"request-rate","value":"a crawl index call is bounded at 5 s by the broker; the listing ends at 8 s","source":null}],"binding":"request-rate","limits_note":"","identity_cost":{"kind":"anonymous","detail":"keyless","country":null},"reliability_note":"504 to a direct index call; 9 consecutive probe timeouts as of 02:50Z 2026-10-22","price_per_tb_month":null,"evidence":[{"url":"https://index.commoncrawl.org/CC-MAIN-2026-38-index?url=example.com&output=json&limit=1","date":"2026-10-22","proved":"504 Gateway Time-out after 11.3 s from nginx: the index did not answer"},{"url":null,"date":"2026-10-22","proved":"state failing, 9 consecutive probe timeouts, the only provider not ok"}],"notes":"Degraded, not dead, and the distinction is the finding: the index answered 504 to a direct call and has failed nine broker probes in a row. A listing can return unknown for every crawl it could not read, so an empty answer is not evidence of absence."},"claims":[{"path":"$.free_tier.stated","kind":"free-quota","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"},{"path":"$.limits[0].value","kind":"limit:request-rate","num":1.0,"unit":"request","per":null,"currency":null,"usd":null,"fx_date":null,"parse":"ok"},{"path":"$.limits[1].value","kind":"limit:request-rate","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"}],"unknown":[["$.evidence[1].url","private-url"],["$.limits[1].source","private-url"],["$.tooling","not-stated"]],"history":[],"drift":[]},{"rid":"storage/common-crawl","card":{"rid":"storage/common-crawl","eid":"commoncrawl","topic":"storage","slug":"common-crawl","provider":"Common Crawl (the petabyte corpus — read side only)","url":"https://commoncrawl.org/get-started","domain":"commoncrawl.org","category":"academic","tier":null,"verdict":"fails-quota","vorder":6,"grp":"rejected","confidence":"reported","binding":"none","identity_kind":"institution","identity_basis":"defaulted","needs_card":null,"needs_phone":null,"needs_id":null,"needs_institution":1,"country":null,"crypto":null,"interfaces":"s3,cli","egress_metered":0,"per_object_bytes":null,"retention_days":null,"verified_at":"2026-10-21T00:00:00.000Z","verified_basis":"inherited","checked_on":"2026-10-21","stale_on":"2026-11-04","n_evidence":3,"usd_month":null,"usd_per_tb_month":null,"summary":"Not a hosting service: ~300 billion web pages across ~100+ monthly crawls, multi-petabyte, free to READ and free to re-host yourself. There is NO path to hos...","duplicate_of":null,"n_unknown":7},"record":{"topic":"storage","slug":"common-crawl","provider":"Common Crawl (the petabyte corpus — read side only)","url":"https://commoncrawl.org/get-started","category":"academic","free_tier":{"stated":"Not a hosting service: ~300 billion web pages across ~100+ monthly crawls, multi-petabyte, free to READ and free to re-host yourself. There is NO path to host your own data with Common Crawl — confirmed, it publishes its own crawls only.","unit":"mixed","metered_on":"unstated","source_url":"https://commoncrawl.org/get-started","read_on":"2026-10-21","notes":null},"per_object_cap":{"stated":"not stated","source_url":"https://commoncrawl.org/get-started","notes":"not stated"},"retention":{"stated":"not stated","source_url":"https://commoncrawl.org/get-started","notes":"not stated"},"egress":{"terms":"Free and anonymous over HTTPS: 'The data is accessible via the https://data.commoncrawl.org/[...] URL scheme. There is no need to create an AWS account in order to access the data using this method.' S3 access (s3://commoncrawl) requires an authenticated AWS account. New: selected crawls are mirrored into a Hugging Face Storage Bucket with CDN pre-warming.","metered":false,"source_url":"https://data.commoncrawl.org/["},"interface":["s3","cli"],"interface_detail":"s3","tooling":"wget/curl against data.commoncrawl.org, aws-cli against s3://commoncrawl (auth), the pycrawl / cc-index-table Athena tooling, and now `hf buckets cp hf://buckets/commoncrawl/commoncrawl/...`.","chaining":"Read-side: any amount of free inbound bandwidth for corpus you would otherwise have to build. The useful pattern for a user is the reverse of storage — Common Crawl is the SOURCE that makes a 200 GB Kaggle/generic pipeline interesting without owning a petabyte.","limits":[],"limits_note":"WARC files are ~1 GB each, ~300k of them; us-east-1 locality strongly recommended ('It's mandatory to access the data from the region where it is located'); the public S3 listing at commoncrawl.s3.amazonaws.com returns 403 to unauthenticated callers (probed) — use HTTPS or HF instead.","identity_cost":{"kind":"institution","detail":"inferred from the category 'academic'; the source record states no identity cost, so this is a class assumption, not evidence","country":null},"reliability_note":"Extremely stable (running since 2008, sponsored by AWS), immutable crawl archives, no access throttling observed on the HTTPS path.","confidence":"reported","confidence_note":"One page read moves it to documented.","evidence":[{"url":"https://commoncrawl.org/get-started","date":"2026-10-21","proved":"'There is no need to create an AWS account in order to access the data using this method'; 'access to data from the Amazon cloud using the S3 API is only allowed for authenticated users'"},{"url":"https://data.commoncrawl.org/crawl-data/CC-MAIN-2026-17/warc.paths.gz","date":"2026-10-21","proved":"probe: GET -> 200, 201,605 bytes, gzip (anonymous listing manifest)"},{"url":"https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2026-17/warc.paths.gz","date":"2026-10-21","proved":"probe: GET -> 403 (S3 path is auth-only)"}],"verdict":"fails-quota","verified_at":"2026-10-21T00:00:00.000Z","notes":"Not a hosting service: ~300 billion web pages across ~100+ monthly crawls, multi-petabyte, free to READ and free to re-host yourself. There is NO path to host your own data with Common Crawl — confirmed, it publishes its own crawls only.","extras":{"dead_end":{"failure_class":"quota","why":"Not a hosting service: ~300 billion web pages across ~100+ monthly crawls, multi-petabyte, free to READ and free to re-host yourself. There is NO path to host your own data with Common Crawl — confirmed, it publishes its own crawls only.","retry_when":null,"first_recorded":"2026-10-21"}},"binding":"none","binding_note":"no constraint of a named kind was recorded; the verdict rests on the quota sentence rather than on a named limit"},"claims":[{"path":"$.free_tier.stated","kind":"free-quota","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"}],"unknown":[["$.extras.dead_end.retry_when","process"],["$.free_tier.notes","process"],["$.identity_cost.kind","identity-defaulted"],["$.per_object_cap.notes","not-extracted"],["$.per_object_cap.stated","not-extracted"],["$.retention.notes","not-extracted"],["$.retention.stated","not-extracted"]],"history":[],"drift":[]}]}