Move initialization call around

ml-agents/mlagents/trainers/common/nn_policy.py

         if self.use_recurrent:
             self.inference_dict["memory_out"] = self.memory_out
 
+        # We do an initialize to make the Policy usable out of the box. If an optimizer is needed,
+        # it will re-load the full graph
+        self._initialize_graph()
+
     @timed
     def evaluate(
         self, batched_step_result: BatchedStepResult, global_agent_ids: List[str]

ml-agents/mlagents/trainers/ppo/optimizer.py

 
             # Add some stuff to inference dict from optimizer
             self.policy.inference_dict["learning_rate"] = self.learning_rate
+            self.policy.initialize_or_load()
 
     def create_cc_critic(
         self, h_size: int, num_layers: int, vis_encode_type: EncoderType
                 "{}_value_estimates".format(name)
             ]
 
-        if "actions_pre" in mini_batch:
+        if self.policy.output_pre is not None and "actions_pre" in mini_batch:
             feed_dict[self.policy.output_pre] = mini_batch["actions_pre"]
         else:
             feed_dict[self.policy.action_holder] = mini_batch["actions"]
             for i, _ in enumerate(self.policy.visual_in):
                 feed_dict[self.policy.visual_in[i]] = mini_batch["visual_obs%d" % i]
         if self.policy.use_recurrent:
-            feed_dict[self.policy.memory_in] = self._make_zero_mem(
-                self.policy.m_size, mini_batch.num_experiences
-            )
+            feed_dict[self.policy.memory_in] = [
+                mini_batch["memory"][i]
+                for i in range(
+                    0, len(mini_batch["memory"]), self.policy.sequence_length
+                )
+            ]
             feed_dict[self.memory_in] = self._make_zero_mem(
                 self.m_size, mini_batch.num_experiences
             )

ml-agents/mlagents/trainers/ppo/trainer.py

             raise RuntimeError("Non-NNPolicy passed to PPOTrainer.add_policy()")
         self.policy = policy
         self.optimizer = PPOOptimizer(self.policy, self.trainer_parameters)
-        self.policy.initialize_or_load()
         for _reward_signal in self.optimizer.reward_signals.keys():
             self.collected_rewards[_reward_signal] = defaultdict(lambda: 0)
         # Needed to resume loads properly

ml-agents/mlagents/trainers/sac/optimizer.py

                         [self.policy.update_normalization_op, target_update_norm]
                     )
 
+                self.policy.initialize_or_load()
+
         self.stats_name_to_update_name = {
             "Losses/Value Loss": "value_loss",
             "Losses/Policy Loss": "policy_loss",

ml-agents/mlagents/trainers/sac/trainer.py

             raise RuntimeError("Non-SACPolicy passed to SACTrainer.add_policy()")
         self.policy = policy
         self.optimizer = SACOptimizer(self.policy, self.trainer_parameters)
-        self.policy.initialize_or_load()
         for _reward_signal in self.optimizer.reward_signals.keys():
             self.collected_rewards[_reward_signal] = defaultdict(lambda: 0)
         # Needed to resume loads properly

ml-agents/mlagents/trainers/tests/test_bcmodule.py

             default_num_epoch=3,
             **trainer_config["behavioral_cloning"],
         )
-    policy.initialize_or_load()
     return bc_module