[tests] Add additional unit tests (#3581)

ml-agents/mlagents/trainers/tests/test_barracuda_converter.py

 import os
 import tempfile
+import pytest
+import yaml
+from mlagents.trainers.tests.test_nn_policy import create_policy_mock
+from mlagents.tf_utils import tf
+from mlagents.model_serialization import SerializationSettings, export_policy_model
 
 
 def test_barracuda_converter():
 
     # cleanup
     os.remove(tmpfile)
+
+
+@pytest.fixture
+def dummy_config():
+    return yaml.safe_load(
+        """
+        trainer: ppo
+        batch_size: 32
+        beta: 5.0e-3
+        buffer_size: 512
+        epsilon: 0.2
+        hidden_units: 128
+        lambd: 0.95
+        learning_rate: 3.0e-4
+        max_steps: 5.0e4
+        normalize: true
+        num_epoch: 5
+        num_layers: 2
+        time_horizon: 64
+        sequence_length: 64
+        summary_freq: 1000
+        use_recurrent: false
+        normalize: true
+        memory_size: 8
+        curiosity_strength: 0.0
+        curiosity_enc_size: 1
+        summary_path: test
+        model_path: test
+        reward_signals:
+          extrinsic:
+            strength: 1.0
+            gamma: 0.99
+        """
+    )
+
+
+@pytest.mark.parametrize("discrete", [True, False], ids=["discrete", "continuous"])
+@pytest.mark.parametrize("visual", [True, False], ids=["visual", "vector"])
+@pytest.mark.parametrize("rnn", [True, False], ids=["rnn", "no_rnn"])
+def test_policy_conversion(dummy_config, tmpdir, rnn, visual, discrete):
+    tf.reset_default_graph()
+    dummy_config["summary_path"] = str(tmpdir)
+    dummy_config["model_path"] = os.path.join(tmpdir, "test")
+    policy = create_policy_mock(
+        dummy_config, use_rnn=rnn, use_discrete=discrete, use_visual=visual
+    )
+    policy.save_model(1000)
+    settings = SerializationSettings(
+        policy.model_path, os.path.join(tmpdir, policy.brain.brain_name)
+    )
+    export_policy_model(settings, policy.graph, policy.sess)
+
+    # These checks taken from test_barracuda_converter
+    assert os.path.isfile(os.path.join(tmpdir, "test.nn"))
+    assert os.path.getsize(os.path.join(tmpdir, "test.nn")) > 100

ml-agents/mlagents/trainers/tests/test_nn_policy.py

     )
 
     trainer_parameters = dummy_config
-    model_path = "testmodel"
-    trainer_parameters["model_path"] = model_path
     trainer_parameters["keep_checkpoints"] = 3
     trainer_parameters["use_recurrent"] = use_rnn
     policy = NNPolicy(0, mock_brain, trainer_parameters, False, False)

ml-agents/mlagents/trainers/tests/test_sac.py

         learning_rate: 3.0e-4
         max_steps: 1024
         memory_size: 10
-        normalize: false
+        normalize: true
         num_update: 1
         train_interval: 1
         num_layers: 1

ml-agents/mlagents/trainers/tests/test_simple_rl.py

     Print to stdout so stats can be viewed in pytest
     """
 
+    def __init__(self):
+        self._last_reward_summary: Dict[str, float] = {}
+
+    def get_last_rewards(self):
+        return self._last_reward_summary
+
     def write_stats(
         self, category: str, values: Dict[str, StatsSummary], step: int
     ) -> None:
+                self._last_reward_summary[category] = stats_summary.mean
 
     def write_text(self, category: str, text: str, step: int) -> None:
         pass
     reward_processor=default_reward_processor,
     meta_curriculum=None,
     success_threshold=0.99,
+    env_manager=None,
 ):
     # Create controller and begin training.
     with tempfile.TemporaryDirectory() as dir:
         StatsReporter.writers.clear()  # Clear StatsReporters so we don't write to file
         debug_writer = DebugWriter()
         StatsReporter.add_writer(debug_writer)
-        env_manager = SimpleEnvManager(env, FloatPropertiesChannel())
+        if env_manager is None:
+            env_manager = SimpleEnvManager(env, FloatPropertiesChannel())
         trainer_factory = TrainerFactory(
             trainer_config=trainer_config,
             summaries_dir=dir,

ml-agents/mlagents/trainers/tests/test_subprocess_env_manager.py

 from unittest import mock
 from unittest.mock import Mock, MagicMock
 import unittest
+import pytest
 from queue import Empty as EmptyQueue
 
 from mlagents.trainers.subprocess_env_manager import (
 from mlagents.trainers.env_manager import EnvironmentStep
 from mlagents_envs.base_env import BaseEnv
 from mlagents_envs.side_channel.engine_configuration_channel import EngineConfig
+from mlagents.trainers.tests.simple_test_envs import Simple1DEnvironment
+from mlagents.trainers.stats import StatsReporter
+from mlagents.trainers.tests.test_simple_rl import (
+    _check_environment_trains,
+    PPO_CONFIG,
+    generate_config,
+    DebugWriter,
+)
 
 
 def mock_env_factory(worker_id):
         self.waiting = False
 
 
+def create_worker_mock(worker_id, step_queue, env_factor, engine_c):
+    return MockEnvWorker(worker_id, EnvironmentResponse("reset", worker_id, worker_id))
+
+
-    def test_environments_are_created(self):
-        SubprocessEnvManager.create_worker = MagicMock()
+    @mock.patch(
+        "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.create_worker"
+    )
+    def test_environments_are_created(self, mock_create_worker):
+        mock_create_worker.side_effect = create_worker_mock
         env = SubprocessEnvManager(mock_env_factory, EngineConfig.default_config(), 2)
         # Creates two processes
         env.create_worker.assert_has_calls(
         )
         self.assertEqual(len(env.env_workers), 2)
 
-    def test_reset_passes_reset_params(self):
-        SubprocessEnvManager.create_worker = lambda em, worker_id, step_queue, env_factory, engine_c: MockEnvWorker(
-            worker_id, EnvironmentResponse("reset", worker_id, worker_id)
-        )
+    @mock.patch(
+        "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.create_worker"
+    )
+    def test_reset_passes_reset_params(self, mock_create_worker):
+        mock_create_worker.side_effect = create_worker_mock
         manager = SubprocessEnvManager(
             mock_env_factory, EngineConfig.default_config(), 1
         )
 
-    def test_reset_collects_results_from_all_envs(self):
-        SubprocessEnvManager.create_worker = lambda em, worker_id, step_queue, env_factory, engine_c: MockEnvWorker(
-            worker_id, EnvironmentResponse("reset", worker_id, worker_id)
-        )
+    @mock.patch(
+        "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.create_worker"
+    )
+    def test_reset_collects_results_from_all_envs(self, mock_create_worker):
+        mock_create_worker.side_effect = create_worker_mock
         manager = SubprocessEnvManager(
             mock_env_factory, EngineConfig.default_config(), 4
         )
             )
         assert res == list(map(lambda ew: ew.previous_step, manager.env_workers))
 
-    def test_step_takes_steps_for_all_non_waiting_envs(self):
-        SubprocessEnvManager.create_worker = lambda em, worker_id, step_queue, env_factory, engine_c: MockEnvWorker(
-            worker_id, EnvironmentResponse("step", worker_id, worker_id)
-        )
+    @mock.patch(
+        "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.create_worker"
+    )
+    def test_step_takes_steps_for_all_non_waiting_envs(self, mock_create_worker):
+        mock_create_worker.side_effect = create_worker_mock
         manager = SubprocessEnvManager(
             mock_env_factory, EngineConfig.default_config(), 3
         )
         "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.external_brains",
         new_callable=mock.PropertyMock,
     )
-    def test_advance(self, external_brains_mock, step_mock):
+    @mock.patch(
+        "mlagents.trainers.subprocess_env_manager.SubprocessEnvManager.create_worker"
+    )
+    def test_advance(self, mock_create_worker, external_brains_mock, step_mock):
-        SubprocessEnvManager.create_worker = lambda em, worker_id, step_queue, env_factory, engine_c: MockEnvWorker(
-            worker_id, EnvironmentResponse("step", worker_id, worker_id)
-        )
+        mock_create_worker.side_effect = create_worker_mock
         env_manager = SubprocessEnvManager(
             mock_env_factory, EngineConfig.default_config(), 3
         )
         env_manager.advance()
         assert env_manager.policies[brain_name] == mock_policy
         assert agent_manager_mock.policy == mock_policy
+
+
+def simple_env_factory(worker_id, config):
+    env = Simple1DEnvironment(["1D"], use_discrete=True)
+    return env
+
+
+@pytest.mark.parametrize("num_envs", [1, 4])
+def test_subprocess_env_endtoend(num_envs):
+    env_manager = SubprocessEnvManager(
+        simple_env_factory, EngineConfig.default_config(), num_envs
+    )
+    trainer_config = generate_config(PPO_CONFIG)
+    # Run PPO using env_manager
+    _check_environment_trains(
+        simple_env_factory(0, []),
+        trainer_config,
+        env_manager=env_manager,
+        success_threshold=None,
+    )
+    # Note we can't check the env's rewards directly (since they're in separate processes) so we
+    # check the StatsReporter's debug stat writer's last reward.
+    assert isinstance(StatsReporter.writers[0], DebugWriter)
+    assert all(
+        val > 0.99 for val in StatsReporter.writers[0].get_last_rewards().values()
+    )
+    env_manager.close()

ml-agents/mlagents/trainers/tests/test_trainer_util.py

 from mlagents.trainers import trainer_util
 from mlagents.trainers.trainer_util import load_config, _load_config
 from mlagents.trainers.ppo.trainer import PPOTrainer
-from mlagents.trainers.exception import TrainerConfigError
+from mlagents.trainers.exception import TrainerConfigError, UnityTrainerException
 from mlagents.trainers.brain import BrainParameters
 
 
     external_brains = {"testbrain": BrainParametersMock()}
 
     with pytest.raises(TrainerConfigError):
+        trainer_factory = trainer_util.TrainerFactory(
+            trainer_config=bad_config,
+            summaries_dir=summaries_dir,
+            run_id=run_id,
+            model_path=model_path,
+            keep_checkpoints=keep_checkpoints,
+            train_model=train_model,
+            load_model=load_model,
+            seed=seed,
+        )
+        trainers = {}
+        for brain_name, brain_parameters in external_brains.items():
+            trainers[brain_name] = trainer_factory.generate(brain_parameters.brain_name)
+
+    # Test no trainer specified
+    del bad_config["default"]["trainer"]
+    with pytest.raises(TrainerConfigError):
+        trainer_factory = trainer_util.TrainerFactory(
+            trainer_config=bad_config,
+            summaries_dir=summaries_dir,
+            run_id=run_id,
+            model_path=model_path,
+            keep_checkpoints=keep_checkpoints,
+            train_model=train_model,
+            load_model=load_model,
+            seed=seed,
+        )
+        trainers = {}
+        for brain_name, brain_parameters in external_brains.items():
+            trainers[brain_name] = trainer_factory.generate(brain_parameters.brain_name)
+
+    # Test BC trainer specified
+    bad_config["default"]["trainer"] = "offline_bc"
+    with pytest.raises(UnityTrainerException):
         trainer_factory = trainer_util.TrainerFactory(
             trainer_config=bad_config,
             summaries_dir=summaries_dir,