config.yaml

### PufferLib demo environments
# Package parameters override defaults.
# Parameters for specific envs override packages
# You cannot specify any deeper than that.
default:
  package: ~
  env_name: ~
  env: {}
  policy: {}
  use_rnn: False
  rnn: {}
  train:
    seed: 1
    torch_deterministic: True
    cpu_offload: False
    device: cuda
    total_timesteps: 10_000_000
    learning_rate: 2.5e-4
    anneal_lr: True
    gamma: 0.99
    gae_lambda: 0.95
    update_epochs: 4
    norm_adv: True
    clip_coef: 0.1
    clip_vloss: True
    vf_coef: 0.5
    vf_clip_coef: 0.1
    max_grad_norm: 0.5
    ent_coef: 0.01
    target_kl: ~

    num_envs: 8
    num_workers: 8
    env_batch_size: ~
    zero_copy: True
    data_dir: experiments
    checkpoint_interval: 200
    batch_size: 1024
    minibatch_size: 512
    bptt_horizon: 16
    compile: False
    compile_mode: reduce-overhead

  sweep:
    method: random
    name: sweep
    metric:
      goal: maximize
      name: environment/episode_return
    # Nested parameters name required by WandB API
    parameters:
      train:
        parameters:
          learning_rate: {
            'distribution': 'log_uniform_values',
            'min': 1e-4,
            'max': 1e-1,
          }
          batch_size: {
            'values': [512, 1024, 2048],
          }
          minibatch_size: {
            'values': [128, 256, 512],
          }
          bptt_horizon: {
            'values': [4, 8, 16],
          }

### Arcade Learning Environment suite
# Convenience wrappers provided for common test environments
atari:
  env_name: BreakoutNoFrameskip-v4
  train:
    batch_size: 1024
    minibatch_size: 256
beamrider:
  package: atari
  env_name: BeamRiderNoFrameskip-v4
beam_rider:
  package: atari
  env_name: BeamRiderNoFrameskip-v4
beam-rider:
  package: atari
  env_name: BeamRiderNoFrameskip-v4
breakout:
  package: atari
  env_name: BreakoutNoFrameskip-v4
enduro:
  package: atari
  env_name: EnduroNoFrameskip-v4
pong:
  package: atari
  env_name: PongNoFrameskip-v4
qbert:
  package: atari
  env_name: QbertNoFrameskip-v4
seaquest:
  package: atari
  env_name: SeaquestNoFrameskip-v4
spaceinvaders:
  package: atari
  env_name: SpaceInvadersNoFrameskip-v4
space_invaders:
  package: atari
  env_name: SpaceInvadersNoFrameskip-v4
space-invaders:
  package: atari
  env_name: SpaceInvadersNoFrameskip-v4

breakout-max-sync:
  package: atari
  env_name: BreakoutNoFrameskip-v4
  train:
    num_envs: 48
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 6144
    minibatch_size: 1536

breakout-max:
  package: atari
  env_name: BreakoutNoFrameskip-v4
  train:
    num_envs: 144 
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 18432
    minibatch_size: 4608

pong-max:
  package: atari
  env_name: PongNoFrameskip-v4
  train:
    num_envs: 96
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 65536
    minibatch_size: 2048


box2d:
  package: box2d

### Procgen Suite
# Shared hyperparams (best for all envs)
# Per-env hyperparams from CARBS
# Note: These need to be updated for 1.0
# batch sizes likely wrong

bigfish-exp:
  package: procgen
  env_name: bigfish
  use_rnn: True
  train:
    total_timesteps: 8_000_000
    num_envs: 480
    num_workers: 24
    env_batch_size: 240
    zero_copy: True
    batch_size: 18432
    minibatch_size: 18432
    update_epochs: 4
    #ent_coef: 0.0025
    #anneal_lr: False

    learning_rate: 0.001
    gamma: 0.9990684264891424
    ent_coef: 0.0025487710400836
    vf_coef: 1.1732211834792117
    gae_lambda: 0.8620630095238284
    clip_coef: 0.4104603426698214
    #batch_size: 53210
    #batch_rows: 5321
    #bptt_horizon: 1
    #update_epochs: 3
    anneal_lr: False
    vf_clip_coef: 0.2

starpilot-exp:
  package: procgen
  env_name: starpilot
  use_rnn: True
  train:
    total_timesteps: 8_000_000
    num_envs: 480
    num_workers: 24
    env_batch_size: 240
    zero_copy: True
    batch_size: 18432
    minibatch_size: 18432
    update_epochs: 4
    learning_rate: 0.0004257280551714
    gamma: 0.9930510505613882
    ent_coef: 0.007836164188961
    vf_coef: 5.482314699746532
    gae_lambda: 0.82792978724664
    clip_coef: 0.2645124138418521
    anneal_lr: False
    vf_clip_coef: 0.2

procgen:
  env_name: bigfish
  train:
    total_timesteps: 25_000_000
    learning_rate: 0.0005
    num_workers: 16
    num_envs: 64
    batch_size: 16384
    minibatch_size: 2048
    gamma: 0.999
    gae_lambda: 0.95
    update_epochs: 3
    anneal_lr: False
    clip_coef: 0.2
    vf_clip_coef: 0.2
bigfish:
  package: procgen
  env_name: bigfish
bossfight:
  package: procgen
  env_name: bossfight
caveflyer:
  package: procgen
  env_name: caveflyer
chaser:
  package: procgen
  env_name: chaser
climber:
  package: procgen
  env_name: climber
coinrun:
  package: procgen
  env_name: coinrun
dodgeball:
  package: procgen
  env_name: dodgeball
fruitbot:
  package: procgen
  env_name: fruitbot
heist:
  package: procgen
  env_name: heist
jumper:
  package: procgen
  env_name: jumper
leaper:
  package: procgen
  env_name: leaper
maze:
  package: procgen
  env_name: maze
miner:
  package: procgen
  env_name: miner
ninja:
  package: procgen
  env_name: ninja
plunder:
  package: procgen
  env_name: plunder
starpilot:
  package: procgen
  env_name: starpilot

bsuite:
  package: bsuite
  env_name: bandit/0
  train:
    total_timesteps: 1_000_000
    num_envs: 1

butterfly:
  package: butterfly
  env_name: cooperative_pong_v5

classic_control:
  env_name: cartpole
  train:
    total_timesteps: 500_000
    num_envs: 64
    env_batch_size: 64
classic-control:
  package: classic_control
classiccontrol:
  package: classic_control
cartpole:
  package: classic_control

crafter:
  package: crafter
  env_name: CrafterReward-v1
  train:
    num_envs: 96
    num_workers: 24
    env_batch_size: 48 
    zero_copy: False
    batch_size: 6144
    compile: False

dm_control:
  package: dm_control
dm-control:
  package: dm_control
dmcontrol:
  package: dm_control
dmc:
  package: dm_control

dm_lab:
  package: dm_lab
dm-lab:
  package: dm_lab
dmlab:
  package: dm_lab
dml:
  package: dm_lab

griddly:
  package: griddly
  env_name: GDY-Spiders-v0

magent:
  package: magent
  env_name: battle_v4

microrts:
  env_name: GlobalAgentCombinedRewardEnv

minerl:
  env_name: MineRLNavigateDense-v0

minigrid:
  env_name: MiniGrid-LavaGapS7-v0
  train:
    total_timesteps: 1_000_000
    num_envs: 48
    num_workers: 6
    env_batch_size: 48
    batch_size: 6144
    #minibatch_size: 768
    update_epochs: 4
    minibatch_size: ~
    #ent_coef: 0.05
    anneal_lr: False
    gae_lambda: 0.95
    gamma: 0.95
    ent_coef: 0.025
    learning_rate: 2.5e-4
  sweep:
    method: bayes
    name: sweep
    metric:
      goal: maximize
      name: environment/episode_return
    # Nested parameters name required by WandB API
    parameters:
      train:
        parameters:
          learning_rate: {
            'distribution': 'log_uniform_values',
            'min': 1e-4,
            'max': 1e-1,
          }
          ent_coef: {
            'distribution': 'log_uniform_values',
            'min': 1e-2,
            'max': 5e-2,
          }
          gamma: {
            'values': [0.90, 0.925, 0.95, 0.975],
          }
          gae_lambda: {
            'values': [0.90, 0.925, 0.95, 0.975],
          }
          batch_size: {
            'values': [384, 768, 1536, 3072, 6144, 12288],
          }
          #minibatch_size: {
          #  'values': [384, 768, 1536],
          #}

minihack:
  env_name: MiniHack-River-v0
  train:
    num_envs: 48
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 6144
    minibatch_size: 1536

nethack:
  env_name: NetHackScore-v0
  train:
    num_envs: 72
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 6144
    update_epochs: 1
    compile: False

nmmo:
  train:
    num_envs: 4
    env_batch_size: 4
    num_workers: 4
    batch_size: 4096
    minibatch_size: 2048

nmmo3:
  use_rnn: True
  train:
    total_timesteps: 100_000_000
    checkpoint_interval: 1000
    num_envs: 24
    num_workers: 24
    env_batch_size: 8
    update_epochs: 1
    gamma: 0.998
    batch_size: 65536
    minibatch_size: 16384
    #compile: True
  env:
    num_envs: 1 # NMMO3 provides its own fast multienv
    #num_envs: 8 # NMMO3 provides its own fast multienv
    #
nmmo3laptop:
  package: nmmo3
  train:
    total_timesteps: 10_000_000
    num_envs: 24
    num_workers: 6
    env_batch_size: 8
    update_epochs: 1
    gamma: 0.998
    batch_size: 32768
    minibatch_size: 16384
    #compile: True
    compile: False

nmmo3debug:
  package: nmmo3
  train:
    total_timesteps: 20_000_000
    num_envs: 1
    num_workers: 1
    env_batch_size: 1
    update_epochs: 1
    gamma: 0.99
    ent_coef: 0.05
    batch_size: 65536
    minibatch_size: 16384
    compile: False
    #anneal_lr: False
  sweep:
    method: random
    name: sweep
    metric:
      goal: maximize
      name: environment/reward_dist
    # Nested parameters name required by WandB API
    parameters:
      train:
        parameters:
          gamma: {
            'values': [0.95, 0.975, 0.99, 0.995, 0.998],
          }
          learning_rate: {
            'distribution': 'log_uniform_values',
            'min': 1e-4,
            'max': 1e-1,
          }
          batch_size: {
            'values': [8192, 16384, 32768],
          }
          bptt_horizon: {
            'values': [4, 8, 16],
          }


nocturne:
  package: nocturne
  env_name: nocturne
  train:
    num_envs: 72
    env_batch_size: 24
    num_workers: 24
    zero_copy: False
    batch_size: 16384
    minibatch_size: 4096
    update_epochs: 1
 
# Ocean: PufferAI's first party environment suite
ocean:
  env_name: squared
  use_rnn: True
  train:
    total_timesteps: 30_000
    learning_rate: 0.017
    num_envs: 8
    num_workers: 2
    env_batch_size: 8
    minibatch_size: 128
    bptt_horizon: 4
    device: cpu
bandit:
  package: ocean
memory:
  package: ocean
multiagent:
  package: ocean
password:
  package: ocean
performance:
  package: ocean
spaces:
  package: ocean
squared:
  package: ocean
stochastic:
  package: ocean

open_spiel:
  env_name: connect_four
  train:
    num_envs: 32
    batch_size: 4096
open-spiel:
  package: open_spiel
openspiel:
  package: open_spiel
connect_four:
  package: open_spiel
  env_name: connect_four
connect-four:
  package: open_spiel
  env_name: connect_four
connectfour:
  package: open_spiel
  env_name: connect_four
connect4:
  package: open_spiel
  env_name: connect_four

pokemon_red:
  use_rnn: True
  train:
    total_timesteps: 1_000_000
    num_envs: 96 
    num_workers: 24
    env_batch_size: 32
    zero_copy: False
    update_epochs: 3
    gamma: 0.998
    batch_size: 65536
    minibatch_size: 2048
    compile: True
    learning_rate: 2.0e-4
    anneal_lr: False
pokemon-red:
  package: pokemon_red
pokemonred:
  package: pokemon_red
pokemon:
  package: pokemon_red
pokegym:
  package: pokemon_red
pokedebug:
  package: pokemon_red
  train:
    num_envs: 4
    num_workers: 4
    env_batch_size: 2
    batch_size: 2048
    minibatch_size: 256
    compile: True
 
links_awaken:
  package: links_awaken
links-awaken:
  package: links_awaken
linksawaken:
  package: links_awaken
zelda:
  package: links_awaken

slimevolley:
  package: slimevolley
  env_name: SlimeVolley-v0
  #use_rnn: True
  train:
    num_envs: 3072
    num_workers: 24
    env_batch_size: 1024
    zero_copy: False
    batch_size: 16384
    minibatch_size: 4096
    update_epochs: 1
  sweep:
    method: bayes
    name: sweep
    metric:
      goal: maximize
      name: environment/episode_return
    # Nested parameters name required by WandB API
    parameters:
      train:
        parameters:
          learning_rate: {
            'distribution': 'log_uniform_values',
            'min': 1e-4,
            'max': 1e-1,
          }
          ent_coef: {
            'distribution': 'log_uniform_values',
            'min': 1e-4,
            'max': 5e-2,
          }
          gamma: {
            'values': [0.90, 0.925, 0.95, 0.975, 0.99],
          }
          gae_lambda: {
            'values': [0.90, 0.925, 0.95, 0.975, 0.99],
          }
          batch_size: {
            'values': [8192, 16384, 32768, 65536],
          }
          minibatch_size: {
            'values': [2048, 4096, 8192],
          }

smac: {}
starcraft:
  package: smac

stable_retro:
  env_name: Airstriker-Genesis
stable-retro:
  package: stable_retro
stableretro:
  package: stable_retro
retro:
  package: stable_retro

vizdoom:
  package: vizdoom
  env_name: VizdoomHealthGatheringSupreme-v0
  use_rnn: True
  train:
    num_envs: 144
    num_workers: 24
    env_batch_size: 48
    zero_copy: False
    batch_size: 8192
    minibatch_size: 2048
    update_epochs: 1