ppo crawler

config/ppo_transfer/CrawlerStatic.yaml

       learning_rate_schedule: linear
       model_schedule: constant
       encoder_layers: 2
-      policy_layers: 2
-      forward_layers: 2
-      value_layers: 3
+      policy_layers: 1
+      forward_layers: 1
+      value_layers: 2
-      reuse_encoder: false
+      separate_model_train: true
+      reuse_encoder: true
-      in_batch_alter: true
-      use_op_buffer: false
+      in_batch_alter: false
+      use_op_buffer: true
-      separate_value_net: true
+      # separate_value_net: true
     network_settings:
       normalize: true
       hidden_units: 512

config/ppo_transfer/TransferCrawlerStatic.yaml

       learning_rate_schedule: linear
       model_schedule: linear
       encoder_layers: 2
-      policy_layers: 2
-      forward_layers: 2
-      value_layers: 3
+      policy_layers: 1
+      forward_layers: 1
+      value_layers: 2
-      reuse_encoder: false
+      separate_model_train: true
+      reuse_encoder: true
-      use_op_buffer: false
+      use_op_buffer: true
-      separate_value_net: true
-      transfer_path: "results/csold-const/CrawlerStatic"
+      load_policy: true
+      load_value: true
+      transfer_path: "results/csold-ppomod/CrawlerStatic"
     network_settings:
       normalize: true
       hidden_units: 512