{"rid":"crawler-scraper/commoncrawl","card":{"rid":"crawler-scraper/commoncrawl","eid":"commoncrawl","topic":"crawler-scraper","slug":"commoncrawl","provider":"Common Crawl","url":"https://commoncrawl.org/","domain":"commoncrawl.org","category":"archive","tier":null,"verdict":"qualifies-with-caveat","vorder":1,"grp":"usable","confidence":"measured","binding":"request-rate","identity_kind":"anonymous","identity_basis":"stated","needs_card":0,"needs_phone":0,"needs_id":0,"needs_institution":0,"country":null,"crypto":null,"interfaces":"rest","egress_metered":0,"per_object_bytes":null,"retention_days":null,"verified_at":"2026-10-22T00:00:00.000Z","verified_basis":"live","checked_on":"2026-10-22","stale_on":"2026-12-21","n_evidence":2,"usd_month":null,"usd_per_tb_month":null,"summary":"keyless index and WARC access","duplicate_of":null,"n_unknown":3},"record":{"topic":"crawler-scraper","slug":"commoncrawl","provider":"Common Crawl","url":"https://commoncrawl.org/","category":"archive","verdict":"qualifies-with-caveat","confidence":"measured","confidence_note":"","verified_at":"2026-10-22T00:00:00.000Z","free_tier":{"stated":"keyless index and WARC access","unit":"requests","metered_on":"request-count","source_url":"https://commoncrawl.org/","read_on":"2026-10-22","notes":""},"per_object_cap":{"stated":"not applicable: the unit is a WARC record","bytes":null,"source_url":"https://commoncrawl.org/","notes":""},"retention":{"stated":"each crawl keeps its own index; up to 12 are asked","days":null,"source_url":"https://commoncrawl.org/","notes":""},"egress":{"terms":"not applicable: the archived body, read from a WARC","metered":false,"free_allowance":null,"price_per_tb":null,"source_url":"https://commoncrawl.org/"},"interface":["rest"],"interface_detail":"Index API per crawl on index.commoncrawl.org; WARC files on data.commoncrawl.org","tooling":"not stated","chaining":"The last archive the broker asks, and the only one that can answer more true on an otherwise empty listing","limits":[{"kind":"request-rate","value":"about 1 request a second on the index, per the platform research report","source":"platform research report 2026-10-20-archive.md"},{"kind":"request-rate","value":"a crawl index call is bounded at 5 s by the broker; the listing ends at 8 s","source":null}],"binding":"request-rate","limits_note":"","identity_cost":{"kind":"anonymous","detail":"keyless","country":null},"reliability_note":"504 to a direct index call; 9 consecutive probe timeouts as of 02:50Z 2026-10-22","price_per_tb_month":null,"evidence":[{"url":"https://index.commoncrawl.org/CC-MAIN-2026-38-index?url=example.com&output=json&limit=1","date":"2026-10-22","proved":"504 Gateway Time-out after 11.3 s from nginx: the index did not answer"},{"url":null,"date":"2026-10-22","proved":"state failing, 9 consecutive probe timeouts, the only provider not ok"}],"notes":"Degraded, not dead, and the distinction is the finding: the index answered 504 to a direct call and has failed nine broker probes in a row. A listing can return unknown for every crawl it could not read, so an empty answer is not evidence of absence."},"claims":[{"path":"$.free_tier.stated","kind":"free-quota","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"},{"path":"$.limits[0].value","kind":"limit:request-rate","num":1,"unit":"request","per":null,"currency":null,"usd":null,"fx_date":null,"parse":"ok"},{"path":"$.limits[1].value","kind":"limit:request-rate","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"}],"unknown":[["$.evidence[1].url","private-url"],["$.limits[1].source","private-url"],["$.tooling","not-stated"]],"history":[],"drift":[]}