[refactor] Structure configuration files into classes (#3936)

com.unity.ml-agents/CHANGELOG.md

 - Curriculum and Parameter Randomization configurations have been merged
   into the main training configuration file. Note that this means training
   configuration files are now environment-specific. (#3791)
+- The format for trainer configuration has changed, and the "default" behavior has been deprecated.
+  See the [Migration Guide](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Migrating.md) for more details. (#3936)
 - Training artifacts (trained models, summaries) are now found in the `results/`
   directory. (#3829)
 - Unity Player logs are now written out to the results directory. (#3877)

config/imitation/CrawlerStatic.yaml

 behaviors:
   CrawlerStatic:
-    trainer: ppo
-    batch_size: 2024
-    beta: 0.005
-    buffer_size: 20240
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    max_steps: 1e7
-    memory_size: 256
-    normalize: true
-    num_epoch: 3
-    num_layers: 3
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
-        strength: 1.0
+        strength: 1.0
+        learning_rate: 0.0003
+        use_actions: false
+        use_vail: false
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true
+      steps: 50000
-      steps: 50000
+      samples_per_update: 0

config/imitation/FoodCollector.yaml

 behaviors:
   FoodCollector:
-    trainer: ppo
-    batch_size: 64
-    beta: 0.005
-    buffer_size: 10240
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    max_steps: 2.0e6
-    memory_size: 256
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 32
-    summary_freq: 10000
-    use_recurrent: false
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 10240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
-        strength: 0.1
+        strength: 0.1
+        learning_rate: 0.0003
+        use_actions: false
+        use_vail: false
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 2000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true
+      steps: 0
-      steps: 0
+      samples_per_update: 0

config/imitation/Hallway.yaml

 behaviors:
   Hallway:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 1024
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    max_steps: 1.0e7
-    memory_size: 256
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: true
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 1024
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+      memory:
+        sequence_length: 64
+        memory_size: 256
-        strength: 1.0
+        strength: 1.0
+        gamma: 0.99
-        gamma: 0.99
+        learning_rate: 0.0003
+        use_actions: false
+        use_vail: false
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true

config/imitation/PushBlock.yaml

 behaviors:
   PushBlock:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    max_steps: 1.5e7
-    memory_size: 256
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 60000
-    use_recurrent: false
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+        learning_rate: 0.0003
+        use_actions: false
+        use_vail: false
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 15000000
+    time_horizon: 64
+    summary_freq: 60000
+    threaded: true

config/imitation/Pyramids.yaml

 behaviors:
   Pyramids:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    max_steps: 1.0e7
-    memory_size: 256
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
+    trainer_type: ppo
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
+    max_steps: 1.0e7
+    hyperparameters:
+      batch_size: 128
+      beta: 0.01
+      buffer_size: 2048
+      epsilon: 0.2
+      lambd: 0.95
+      learning_rate: 0.0003
+      num_epoch: 3
+    network_settings:
+      num_layers: 2
+      normalize: false
+      hidden_units: 512
     reward_signals:
       extrinsic:
         strength: 1.0

config/ppo/3DBall.yaml

 behaviors:
   3DBall:
-    trainer: ppo
-    batch_size: 64
-    beta: 0.001
-    buffer_size: 12000
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.99
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 5.0e5
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 12000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 12000
+      learning_rate: 0.0003
+      beta: 0.001
+      epsilon: 0.2
+      lambd: 0.99
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 500000
+    time_horizon: 1000
+    summary_freq: 12000
+    threaded: true

config/ppo/3DBallHard.yaml

 behaviors:
   3DBallHard:
-    trainer: ppo
-    batch_size: 1200
-    beta: 0.001
-    buffer_size: 12000
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 5.0e6
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 12000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 1200
+      buffer_size: 12000
+      learning_rate: 0.0003
+      beta: 0.001
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 5000000
+    time_horizon: 1000
+    summary_freq: 12000
+    threaded: true

config/ppo/3DBall_randomize.yaml

 behaviors:
-    3DBall:
-        trainer: ppo
-        batch_size: 64
-        beta: 0.001
-        buffer_size: 12000
-        epsilon: 0.2
-        hidden_units: 128
-        lambd: 0.99
-        learning_rate: 3.0e-4
-        learning_rate_schedule: linear
-        max_steps: 5.0e5
-        memory_size: 128
-        normalize: true
-        num_epoch: 3
-        num_layers: 2
-        time_horizon: 1000
-        sequence_length: 64
-        summary_freq: 12000
-        use_recurrent: false
-        vis_encode_type: simple
-        reward_signals:
-            extrinsic:
-                strength: 1.0
-                gamma: 0.99
+  3DBall:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 12000
+      learning_rate: 0.0003
+      beta: 0.001
+      epsilon: 0.2
+      lambd: 0.99
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+    reward_signals:
+      extrinsic:
+        gamma: 0.99
+        strength: 1.0
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 500000
+    time_horizon: 1000
+    summary_freq: 12000
+    threaded: true
-    resampling-interval: 5000
-    mass:
-        sampler-type: "uniform"
-        min_value: 0.5
-        max_value: 10
-    gravity:
-        sampler-type: "uniform"
-        min_value: 7
-        max_value: 12
-    scale:
-        sampler-type: "uniform"
-        min_value: 0.75
-        max_value: 3
+  resampling-interval: 5000
+  mass:
+    sampler-type: uniform
+    min_value: 0.5
+    max_value: 10
+  gravity:
+    sampler-type: uniform
+    min_value: 7
+    max_value: 12
+  scale:
+    sampler-type: uniform
+    min_value: 0.75
+    max_value: 3

config/ppo/Basic.yaml

 behaviors:
   Basic:
-    trainer: ppo
-    batch_size: 32
-    beta: 0.005
-    buffer_size: 256
-    epsilon: 0.2
-    hidden_units: 20
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 5.0e5
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 1
-    time_horizon: 3
-    sequence_length: 64
-    summary_freq: 2000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 32
+      buffer_size: 256
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 20
+      num_layers: 1
+      vis_encode_type: simple
+        gamma: 0.9
-        gamma: 0.9
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 500000
+    time_horizon: 3
+    summary_freq: 2000
+    threaded: true

config/ppo/Bouncer.yaml

 behaviors:
   Bouncer:
-    trainer: ppo
-    batch_size: 1024
-    beta: 0.005
-    buffer_size: 10240
-    epsilon: 0.2
-    hidden_units: 64
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 4.0e6
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 1024
+      buffer_size: 10240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 64
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 4000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true

config/ppo/CrawlerDynamic.yaml

 behaviors:
   CrawlerDynamic:
-    trainer: ppo
-    batch_size: 2024
-    beta: 0.005
-    buffer_size: 20240
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1e7
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 3
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true

config/ppo/CrawlerStatic.yaml

 behaviors:
   CrawlerStatic:
-    trainer: ppo
-    batch_size: 2024
-    beta: 0.005
-    buffer_size: 20240
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1e7
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 3
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true

config/ppo/FoodCollector.yaml

 behaviors:
   FoodCollector:
-    trainer: ppo
-    batch_size: 1024
-    beta: 0.005
-    buffer_size: 10240
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2.0e6
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 1024
+      buffer_size: 10240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 2000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true

config/ppo/GridWorld.yaml

 behaviors:
   GridWorld:
-    trainer: ppo
-    batch_size: 32
-    beta: 0.005
-    buffer_size: 256
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 500000
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 1
-    time_horizon: 5
-    sequence_length: 64
-    summary_freq: 20000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 32
+      buffer_size: 256
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 1
+      vis_encode_type: simple
+        gamma: 0.9
-        gamma: 0.9
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 500000
+    time_horizon: 5
+    summary_freq: 20000
+    threaded: true

config/ppo/Hallway.yaml

 behaviors:
   Hallway:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 1024
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: true
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 1024
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+      memory:
+        sequence_length: 64
+        memory_size: 128
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true

config/ppo/PushBlock.yaml

 behaviors:
   PushBlock:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2.0e6
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 60000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 2000000
+    time_horizon: 64
+    summary_freq: 60000
+    threaded: true

config/ppo/Pyramids.yaml

 behaviors:
   Pyramids:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.01
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 128
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
-        strength: 1.0
+        strength: 1.0
-        strength: 0.02
+        strength: 0.02
+        learning_rate: 0.0003
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 128
+    summary_freq: 30000
+    threaded: true

config/ppo/Reacher.yaml

 behaviors:
   Reacher:
-    trainer: ppo
-    batch_size: 2024
-    beta: 0.005
-    buffer_size: 20240
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2e7
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 60000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 128
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 20000000
+    time_horizon: 1000
+    summary_freq: 60000
+    threaded: true

config/ppo/SoccerTwos.yaml

 behaviors:
   SoccerTwos:
-    trainer: ppo
-    batch_size: 2048
-    beta: 0.005
-    buffer_size: 20480
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: constant
-    max_steps: 5.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2048
+      buffer_size: 20480
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: constant
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
-        strength: 1.0
+        strength: 1.0
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 50000000
+    time_horizon: 1000
+    summary_freq: 10000
+    threaded: true
-      window: 10
-      play_against_latest_model_ratio: 0.5
-      swap_steps: 50000
-    curriculum:
-      measure: progress
-      thresholds: [0.05, 0.1]
-      min_lesson_length: 100
-      signal_smoothing: true
-      parameters:
-        ball_touch: [1.0, 0.5, 0.0]
+      swap_steps: 50000
+      window: 10
+      play_against_latest_model_ratio: 0.5
+      initial_elo: 1200.0

config/ppo/StrikersVsGoalie.yaml

 behaviors:
   Goalie:
-    trainer: ppo
-    batch_size: 2048
-    beta: 0.005
-    buffer_size: 20480
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: constant
-    max_steps: 5.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2048
+      buffer_size: 20480
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: constant
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 50000000
+    time_horizon: 1000
+    summary_freq: 10000
+    threaded: true
-      window: 10
-      play_against_latest_model_ratio: 0.5
-      swap_steps: 25000
-
+      swap_steps: 25000
+      window: 10
+      play_against_latest_model_ratio: 0.5
+      initial_elo: 1200.0
-    trainer: ppo
-    batch_size: 2048
-    beta: 0.005
-    buffer_size: 20480
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: constant
-    max_steps: 5.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2048
+      buffer_size: 20480
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: constant
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 50000000
+    time_horizon: 1000
+    summary_freq: 10000
+    threaded: true
-      window: 10
-      play_against_latest_model_ratio: 0.5
+      team_change: 200000
-      team_change: 200000
+      window: 10
+      play_against_latest_model_ratio: 0.5
+      initial_elo: 1200.0

config/ppo/Tennis.yaml

 behaviors:
   Tennis:
-    trainer: ppo
-    batch_size: 1024
-    beta: 0.005
-    buffer_size: 10240
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: constant
-    max_steps: 5.0e7
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 1024
+      buffer_size: 10240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: constant
+    network_settings:
+      normalize: true
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 50000000
+    time_horizon: 1000
+    summary_freq: 10000
+    threaded: true
-      window: 10
-      play_against_latest_model_ratio: 0.5
+      team_change: 100000
-      team_change: 100000
+      window: 10
+      play_against_latest_model_ratio: 0.5
+      initial_elo: 1200.0

config/ppo/VisualHallway.yaml

 behaviors:
   VisualHallway:
-    trainer: ppo
-    batch_size: 64
-    beta: 0.01
-    buffer_size: 1024
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 1
-    time_horizon: 64
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: true
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 1024
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 1
+      vis_encode_type: simple
+      memory:
+        sequence_length: 64
+        memory_size: 128
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 64
+    summary_freq: 10000
+    threaded: true

config/ppo/VisualPushBlock.yaml

 behaviors:
   VisualPushBlock:
-    trainer: ppo
-    batch_size: 64
-    beta: 0.01
-    buffer_size: 1024
-    epsilon: 0.2
-    hidden_units: 128
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 3.0e6
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 1
-    time_horizon: 64
-    sequence_length: 32
-    summary_freq: 60000
-    use_recurrent: true
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 1024
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 1
+      vis_encode_type: simple
+      memory:
+        sequence_length: 32
+        memory_size: 128
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 3000000
+    time_horizon: 64
+    summary_freq: 60000
+    threaded: true

config/ppo/VisualPyramids.yaml

 behaviors:
   VisualPyramids:
-    trainer: ppo
-    batch_size: 64
-    beta: 0.01
-    buffer_size: 2024
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 1.0e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 1
-    time_horizon: 128
-    sequence_length: 64
-    summary_freq: 10000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 64
+      buffer_size: 2024
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 1
+      vis_encode_type: simple
-        strength: 1.0
+        strength: 1.0
-        strength: 0.01
+        strength: 0.01
+        learning_rate: 0.0003
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 10000000
+    time_horizon: 128
+    summary_freq: 10000
+    threaded: true

config/ppo/Walker.yaml

 behaviors:
   Walker:
-    trainer: ppo
-    batch_size: 2048
-    beta: 0.005
-    buffer_size: 20480
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2e7
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 3
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2048
+      buffer_size: 20480
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 20000000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true

config/ppo/WallJump.yaml

 behaviors:
   BigWallJump:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.005
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 128
-    sequence_length: 64
-    summary_freq: 20000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
-        strength: 1.0
-
-  SmallWallJump:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.005
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 5e6
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
+        strength: 1.0
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 20000000
-    sequence_length: 64
-    use_recurrent: false
-    vis_encode_type: simple
+    threaded: true
+  SmallWallJump:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 5000000
+    time_horizon: 128
+    summary_freq: 20000
+    threaded: true

config/ppo/WallJump_curriculum.yaml

 behaviors:
   BigWallJump:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.005
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 2e7
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
-    time_horizon: 128
-    sequence_length: 64
-    summary_freq: 20000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
-    curriculum:
-      measure: progress
-      thresholds: [0.1, 0.3, 0.5]
-      min_lesson_length: 100
-      signal_smoothing: true
-      parameters:
-        big_wall_min_height: [0.0, 4.0, 6.0, 8.0]
-        big_wall_max_height: [4.0, 7.0, 8.0, 8.0]
-
-  SmallWallJump:
-    trainer: ppo
-    batch_size: 128
-    beta: 0.005
-    buffer_size: 2048
-    epsilon: 0.2
-    hidden_units: 256
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 5e6
-    memory_size: 128
-    normalize: false
-    num_epoch: 3
-    num_layers: 2
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 20000000
-    sequence_length: 64
-    use_recurrent: false
-    vis_encode_type: simple
+    threaded: true
+  SmallWallJump:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 256
+      num_layers: 2
+      vis_encode_type: simple
+        gamma: 0.99
-        gamma: 0.99
-    curriculum:
-      measure: progress
-      thresholds: [0.1, 0.3, 0.5]
-      min_lesson_length: 100
-      signal_smoothing: true
-      parameters:
-        small_wall_height: [1.5, 2.0, 2.5, 4.0]
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 5000000
+    time_horizon: 128
+    summary_freq: 20000
+    threaded: true
+
+curriculum:
+  BigWallJump:
+    measure: progress
+    thresholds: [0.1, 0.3, 0.5]
+    min_lesson_length: 100
+    signal_smoothing: true
+    parameters:
+      big_wall_min_height: [0.0, 4.0, 6.0, 8.0]
+      big_wall_max_height: [4.0, 7.0, 8.0, 8.0]
+  SmallWallJump:
+    measure: progress
+    thresholds: [0.1, 0.3, 0.5]
+    min_lesson_length: 100
+    signal_smoothing: true
+    parameters:
+      small_wall_height: [1.5, 2.0, 2.5, 4.0]

config/ppo/WormDynamic.yaml

 behaviors:
   WormDynamic:
-    trainer: ppo
-    batch_size: 2024
-    beta: 0.005
-    buffer_size: 20240
-    epsilon: 0.2
-    hidden_units: 512
-    lambd: 0.95
-    learning_rate: 0.0003
-    learning_rate_schedule: linear
-    max_steps: 3.5e6
-    memory_size: 128
-    normalize: true
-    num_epoch: 3
-    num_layers: 3
-    time_horizon: 1000
-    sequence_length: 64
-    summary_freq: 30000
-    use_recurrent: false
-    vis_encode_type: simple
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+        gamma: 0.995
-        gamma: 0.995
+    output_path: default
+    keep_checkpoints: 5
+    max_steps: 3500000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true