model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.004-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.004-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.004-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.004-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.004-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.004-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.004-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.004-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.004
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.004-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.005-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.005-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.005-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.005-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.005-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.005-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.005-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.005-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.005
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.005-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.006-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.006-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.006-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.006-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.006-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.006-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.006-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.006-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.006
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.006-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.007-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.007-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.007-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.007-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.007-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.007-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.007-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.007-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.007
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.007-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.008-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.008-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.008-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.008-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.008-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.008-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.008-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.008-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.008
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.008-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.009-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.009-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.009-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.009-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.009-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.009-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.009-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.009-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.009
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.009-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.01-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.01-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.01-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.01-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.01-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.99
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.99-lr-0.01-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.01-s42/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 43
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.01-s43/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.01
  beta1: 0.9
  beta2: 0.999
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 44
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion/experiments/horizon-20/b1-0.9-b2-0.999-lr-0.01-s44/attempt-001
  cpu_threads: 8
decentralized: null


model:
  vocab_size: 32000
  layers: 8
  width: 320
  heads: 5
  ffn_width: 896
  context_length: 1024
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  init_std: 0.02
data:
  cache_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/data/c4
  dataset: allenai/c4
  revision: 1588ec454efa1a09f29cd18ddd04fe05fc8653a2
  tokenizer: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
  tokenizer_revision: 59f6f375b26bde864a6ca194a9a3044570490064
  shuffle_seed: 42
  shuffle_buffer: 100000
  shard_tokens: 33554432
  tokenize_batch_size: 256
  prepare_workers: 8
  shuffle_group_size: 2
  prefetch: true
  prepare_train_tokens: null
  prepare_validation_tokens: null
optimizer:
  lr: 0.011
  beta1: 0.9
  beta2: 0.98
  eps: 1.0e-08
  weight_decay: 0.1
  grad_clip: 1.0
lr_schedule:
  name: cosine
  warmup_steps: 312
  min_lr_ratio: 0.0
training:
  tokens_per_parameters: 20.0
  epoch_tokens_per_parameters: 0.5
  batch_tokens: 131072
  micro_batch_size: 128
  checkpoint_policy: none
  checkpoint_epochs:
  - 1
  save_epoch_training_state: false
  log_every: 20
evaluation:
  subset_blocks: 1024
  batch_size: 128
  seed: 12345
runtime:
  seed: 42
  deterministic: false
  device: cuda:0
  amp: true
  compile: true
  compile_mode: default
  sdpa_backend: auto
  fused_optimizer: true
  attention_backend: sdpa
  output_dir: /nobackup/proj/disk/naiss2024-23-12/personal/zesen/repo/tiny-llm/runs/sync-cosine-zero-20m-horizon-tuning/extensions/lr-expansion-2/experiments/horizon-20/b1-0.9-b2-0.98-lr-0.011-s42/attempt-001
  cpu_threads: 8
decentralized: null