new transfer test for cloud

ml-agents/mlagents/trainers/learn.py

         run_seed = np.random.randint(0, 10000)
     run_training(run_seed, options)
     
-    if options.behaviors["CrawlerStatic"].transfer:
-        os.system('export SCENE_NAME=crawlerstatictarget')
-        os.system('mlagents-learn config/ppo_transfer/CrawlerStatic.yaml --run-id=cs-transfer --env=/unity-volume/executable --num-envs=4 --force')
+    # if options.behaviors["CrawlerStatic"].transfer:
+    #     os.system('export SCENE_NAME=crawlerstatictarget')
+    #     os.system('mlagents-learn config/ppo_transfer/CrawlerStatic.yaml --run-id=cs-transfer --env=/unity-volume/executable --num-envs=4 --force')
     #     os.system('mlagents-learn config/ppo_transfer/3DBallHard.yaml --run-id=hardball-transfer --env=/unity-volume/3dballhard --num-envs=4 --force')
 
 

ml-agents/mlagents/trainers/policy/transfer_policy.py

         # slim.model_analyzer.analyze_vars(self.trainable_variables, print_info=True)
     
     def load_graph_partial(self, path: str, transfer_type="dynamics"):
-        load_nets = {"dynamics": ["policy", "predict", "value"], 
+        load_nets = {"dynamics": ["predict"], 
             "observation": ["encoding", "inverse"]}
         if self.inverse_model:
             load_nets["dynamics"].append("inverse")
                     hidden_stream_targ,
                     feature_size,
                     name="latent",
-                    reuse=reuse_encoder
+                    reuse=reuse_encoder,
+                    # activation=ModelUtils.swish,
+                    # kernel_initializer=tf.initializers.variance_scaling(1.0),
                 )
         return latent_targ
         # return tf.stop_gradient(latent_targ)
             latent = tf.layers.dense(
                     hidden_stream,
                     feature_size,
-                    name="latent"
+                    name="latent",
+                    # activation=ModelUtils.swish,
+                    # kernel_initializer=tf.initializers.variance_scaling(1.0),
                 )
         return latent
     
                 hidden,
                 self.h_size
                 * (self.vis_obs_size + int(self.vec_obs_size > 0)),
-                activation=None,
-                name="hidden_{}".format(i)
+                name="hidden_{}".format(i),
+                # activation=ModelUtils.swish,
+                # kernel_initializer=tf.initializers.variance_scaling(1.0),
             )
 
         if var_predict:
             self.predict = tf.layers.dense(
                 hidden,
                 self.feature_size,
-                name="latent"
+                name="latent",
+                # activation=ModelUtils.swish,
+                # kernel_initializer=tf.initializers.variance_scaling(1.0),
             )
 
         squared_difference = 0.5 * tf.reduce_sum(
             self.pred_reward = tf.layers.dense(
                 hidden,
                 1,
-                name="reward"
+                name="reward",
+                activation=ModelUtils.swish,
+                kernel_initializer=tf.initializers.variance_scaling(1.0),
             )
             self.forward_loss += tf.reduce_mean(
                 tf.dynamic_partition(

ml-agents/mlagents/trainers/ppo_transfer/optimizer.py

 
             with tf.variable_scope("value"):
                 if policy.use_continuous_act:
-                    self._create_cc_critic(h_size, hyperparameters.value_layers, vis_encode_type)
+                    self._create_cc_critic_old(h_size, hyperparameters.value_layers, vis_encode_type)
                 else:
                     self._create_dc_critic(h_size, hyperparameters.value_layers, vis_encode_type)
             

config/ppo_transfer/OldCrawlerStaticTest.yaml

+behaviors:
+  CrawlerStatic:
+    trainer_type: ppo_transfer
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+      encoder_layers: 2
+      policy_layers: 2
+      value_layers: 2
+      feature_size: 32
+      reuse_encoder: true
+      in_epoch_alter: true
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+    reward_signals:
+      extrinsic:
+        gamma: 0.995
+        strength: 1.0
+    keep_checkpoints: 5
+    max_steps: 100000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true

config/ppo_transfer/TransferCrawlerStaticTest.yaml

+behaviors:
+  CrawlerStatic:
+    trainer_type: ppo_transfer
+    hyperparameters:
+      batch_size: 2024
+      buffer_size: 20240
+      learning_rate: 0.0003
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      learning_rate_schedule: linear
+      encoder_layers: 2
+      policy_layers: 2
+      value_layers: 2
+      feature_size: 32
+      reuse_encoder: true
+      in_epoch_alter: true
+      use_transfer: true
+      train_type: "policy"
+      transfer_path: "results/cs-old-single/CrawlerStatic"
+    network_settings:
+      normalize: true
+      hidden_units: 512
+      num_layers: 3
+      vis_encode_type: simple
+    reward_signals:
+      extrinsic:
+        gamma: 0.995
+        strength: 1.0
+    keep_checkpoints: 5
+    max_steps: 100000
+    time_horizon: 1000
+    summary_freq: 30000
+    threaded: true
+    transfer: true