model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 10000
  shard_tokens: 16777216
  tokenize_batch_size: 256
  buffer_size_mib: 64.0
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.0048
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
  warmup_fraction: 0.05
  min_lr_ratio: 0.1
training:
  tokens_per_parameter: 20.0
  epoch_tokens_per_parameter: 0.5
  batch_tokens: 32768
  micro_batch_size: 8
  checkpoint_every: 500
  checkpoint_policy: final
  log_every: 20
  max_tokens: 408068096
  epoch_tokens: 10207232
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: runs/20m-selected-packed-4
  cpu_threads: 8
decentralized:
  num_models: 4
  topology: one_peer_ring