{
  "schema_version": 1,
  "format": "teutonic-packed-text-inventory-v1",
  "dataset": {
    "bucket": "dataset",
    "key": "teutonic-text/manifest.json",
    "sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
  },
  "dataset_label": "Full packed-text dataset",
  "token_count_basis": "original_mimo_token_count",
  "sources": [
    {
      "name": "code-no-thinking",
      "dataset": "code-reasoning",
      "total_tokens": 462582544925,
      "total_documents": 131252301,
      "total_shards": 13269,
      "total_utf8_bytes": 1756657504232,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "code-no-thinking-hard",
      "dataset": "code-reasoning",
      "total_tokens": 360717737100,
      "total_documents": 79282021,
      "total_shards": 10765,
      "total_utf8_bytes": 1423836214663,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "code-task-synthesis",
      "dataset": "code-reasoning",
      "total_tokens": 292237333758,
      "total_documents": 49028304,
      "total_shards": 9084,
      "total_utf8_bytes": 1198594499521,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "code-thinking-hard",
      "dataset": "code-reasoning",
      "total_tokens": 300230065894,
      "total_documents": 34348121,
      "total_shards": 8769,
      "total_utf8_bytes": 1176178436031,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "code-thinking-v0",
      "dataset": "code-reasoning",
      "total_tokens": 523955250720,
      "total_documents": 82212487,
      "total_shards": 15828,
      "total_utf8_bytes": 2098060266335,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "code-thinking-v1",
      "dataset": "code-reasoning",
      "total_tokens": 488813831716,
      "total_documents": 66294120,
      "total_shards": 14130,
      "total_utf8_bytes": 1856537639189,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "codeio",
      "dataset": "code-reasoning",
      "total_tokens": 24682860219,
      "total_documents": 9128884,
      "total_shards": 621,
      "total_utf8_bytes": 82592530527,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "dclm-baseline-1.0",
      "dataset": "dclm-baseline-1.0",
      "total_tokens": 3732171112203,
      "total_documents": 2939200006,
      "total_shards": 135983,
      "total_utf8_bytes": 16624726668030,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "math-dialogue",
      "dataset": "math-reasoning",
      "total_tokens": 1932428105278,
      "total_documents": 1261369146,
      "total_shards": 46445,
      "total_utf8_bytes": 6186058509650,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "math-rewrite",
      "dataset": "math-reasoning",
      "total_tokens": 654783926019,
      "total_documents": 251859206,
      "total_shards": 16137,
      "total_utf8_bytes": 2129856700164,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "math-thinking-oss",
      "dataset": "math-reasoning",
      "total_tokens": 487488070643,
      "total_documents": 247660864,
      "total_shards": 9472,
      "total_utf8_bytes": 1249915119596,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "math-thinking-qwen",
      "dataset": "math-reasoning",
      "total_tokens": 811029829991,
      "total_documents": 249291953,
      "total_shards": 16973,
      "total_utf8_bytes": 2240760091399,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "socratic-math-dialogue",
      "dataset": "math-reasoning",
      "total_tokens": 157754466280,
      "total_documents": 5897981,
      "total_shards": 2170,
      "total_utf8_bytes": 285420451375,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "data-science",
      "dataset": "pretrain-behaviors",
      "total_tokens": 448671541575,
      "total_documents": 123487382,
      "total_shards": 13893,
      "total_utf8_bytes": 1831482348544,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "format-rewrites",
      "dataset": "pretrain-behaviors",
      "total_tokens": 376221725022,
      "total_documents": 256019384,
      "total_shards": 12743,
      "total_utf8_bytes": 1694991571142,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "games",
      "dataset": "pretrain-behaviors",
      "total_tokens": 168704535339,
      "total_documents": 65385518,
      "total_shards": 5983,
      "total_utf8_bytes": 796112257348,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "general",
      "dataset": "pretrain-behaviors",
      "total_tokens": 684388618494,
      "total_documents": 206982151,
      "total_shards": 23328,
      "total_utf8_bytes": 3052965780863,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "other",
      "dataset": "pretrain-behaviors",
      "total_tokens": 519580595204,
      "total_documents": 191542392,
      "total_shards": 16965,
      "total_utf8_bytes": 2252953243194,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "planning",
      "dataset": "pretrain-behaviors",
      "total_tokens": 938289528254,
      "total_documents": 289779117,
      "total_shards": 29152,
      "total_utf8_bytes": 3866948869472,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "reasoning",
      "dataset": "pretrain-behaviors",
      "total_tokens": 1558228823490,
      "total_documents": 501833026,
      "total_shards": 50330,
      "total_utf8_bytes": 6751734563549,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "3efforts-pretrain",
      "dataset": "sft-reasoning",
      "total_tokens": 82360695471,
      "total_documents": 32215646,
      "total_shards": 2346,
      "total_utf8_bytes": 308488716028,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "instruction-following",
      "dataset": "sft-reasoning",
      "total_tokens": 46262754214,
      "total_documents": 8581736,
      "total_shards": 1497,
      "total_utf8_bytes": 199673516717,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "txt360-qa",
      "dataset": "txt360-v2",
      "total_tokens": 1275805060942,
      "total_documents": 581171081,
      "total_shards": 44078,
      "total_utf8_bytes": 5894993331253,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "web-high-medium",
      "dataset": "txt360-v2",
      "total_tokens": 899935001580,
      "total_documents": 770470878,
      "total_shards": 29769,
      "total_utf8_bytes": 3914438040820,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    },
    {
      "name": "web-high-nltk-qa",
      "dataset": "txt360-v2",
      "total_tokens": 948425959846,
      "total_documents": 484376625,
      "total_shards": 30439,
      "total_utf8_bytes": 4060239934186,
      "tokenization_mode": "packed-text",
      "dtype": "utf-8",
      "manifest_url": "s3://dataset/teutonic-text/manifest.json",
      "manifest_sha256": "58ce39daf1a3b5be5860779faaf32245d1d040edad25c9abf17c399369e96448"
    }
  ]
}