{"eid":"datalad","name":"DataLad + git-annex workflow","domain":"datalad.org","resolved_by":"domain","topics":["storage"],"n_records":1,"best_vorder":1,"review":null,"aliases":["DataLad + git-annex workflow","DataLad + git-annex workflow (orchestration layer)","datalad.org"],"records":[{"rid":"storage/datalad-git-annex-workflow","card":{"rid":"storage/datalad-git-annex-workflow","eid":"datalad","topic":"storage","slug":"datalad-git-annex-workflow","provider":"DataLad + git-annex workflow (orchestration layer)","url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","domain":"datalad.org","category":"academic","tier":null,"verdict":"qualifies-with-caveat","vorder":1,"grp":"usable","confidence":"reported","binding":"none","identity_kind":"institution","identity_basis":"defaulted","needs_card":null,"needs_phone":null,"needs_id":null,"needs_institution":1,"country":null,"crypto":null,"interfaces":"s3,git,cli","egress_metered":0,"per_object_bytes":null,"retention_days":null,"verified_at":"2026-10-21T00:00:00.000Z","verified_basis":"inherited","checked_on":"2026-10-21","stale_on":"2026-11-04","n_evidence":2,"usd_month":null,"usd_per_tb_month":null,"summary":"N/A — DataLad is a wrapper over git-annex; quota is the special remote's. Its value here is that it makes 'dataset + N special remotes + reproducible publish...","duplicate_of":null,"n_unknown":6},"record":{"topic":"storage","slug":"datalad-git-annex-workflow","provider":"DataLad + git-annex workflow (orchestration layer)","url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","category":"academic","free_tier":{"stated":"N/A — DataLad is a wrapper over git-annex; quota is the special remote's. Its value here is that it makes 'dataset + N special remotes + reproducible publish' a supported, documented workflow instead of hand-written annex commands.","unit":"mixed","metered_on":"unstated","source_url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","read_on":"2026-10-21","notes":null},"per_object_cap":{"stated":"not stated","source_url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","notes":"not stated"},"retention":{"stated":"not stated","source_url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","notes":"not stated"},"egress":{"terms":"Inherited; DataLad's S3 walkthrough specifically configures `publicurl` so consumers clone the git metadata from GitHub and stream file content from the bucket with no credentials.","metered":false},"interface":["s3","git","cli"],"interface_detail":"s3","tooling":"`datalad` CLI (Python), `datalad create-sibling-s3`, `datalad push --to <sibling>`, `datalad get`. Requires git-annex >= 10.20230802 for S3 publicurl on post-April-2023 buckets.","chaining":"DataLad's 'dataset hierarchy' (superdataset + subdatasets) maps a 20 TiB archive into hundreds of independently-served subdatasets, each with its own sibling/remote. Combined with `datalad siblings` this is the cleanest way to shard one logical corpus across many free S3 accounts while keeping one coherent git history.","limits":[],"limits_note":"Same as annex: no delete on LFS-backed remotes, per-store object caps. DataLad adds Python overhead and its own version-compat matrix (annex version pinned per feature).","identity_cost":{"kind":"institution","detail":"inferred from the category 'academic'; the source record states no identity cost, so this is a class assumption, not evidence","country":null},"reliability_note":"Used in production by academic consortia (OpenNeuro etc.) precisely for multi-TB datasets on donated storage — the most battle-tested 'big dataset on free/cheap stores' pattern that exists.","confidence":"reported","confidence_note":"One page read moves it to documented.","evidence":[{"url":"https://handbook.datalad.org/en/latest/basics/101-139-s3.html","date":"2026-10-21","proved":"'This walk-through requires git-annex >= 10.20230802 ... Prior versions do not support public access via the publicurl parameter with S3 buckets created after April 2023'"},{"url":"https://handbook.datalad.org/en/latest/basics/101-139-gitlfs.html","date":"2026-10-21","proved":"DataLad's Git LFS special-remote walkthrough on GitHub (linked as next chapter)"}],"verdict":"qualifies-with-caveat","verified_at":"2026-10-21T00:00:00.000Z","notes":"N/A — DataLad is a wrapper over git-annex; quota is the special remote's. Its value here is that it makes 'dataset + N special remotes + reproducible publish' a supported, documented workflow instead of hand-written annex commands.","extras":{},"binding":"none","binding_note":"no constraint in the record binds at this scale; the record carries no limit of a kind the vocabulary names"},"claims":[{"path":"$.free_tier.stated","kind":"free-quota","num":null,"unit":null,"per":null,"currency":null,"usd":null,"fx_date":null,"parse":"unparsed"}],"unknown":[["$.free_tier.notes","process"],["$.identity_cost.kind","identity-defaulted"],["$.per_object_cap.notes","not-extracted"],["$.per_object_cap.stated","not-extracted"],["$.retention.notes","not-extracted"],["$.retention.stated","not-extracted"]],"history":[],"drift":[]}]}