浏览代码

Merge branch 'develop-add-fire' into develop-add-fire-checkpoint

/develop/add-fire/ckpt-2
Ruo-Ping Dong 4 年前
当前提交
59cc1a9f
共有 17 个文件被更改,包括 2091 次插入144 次删除
  1. 63
      ml-agents/mlagents/trainers/optimizer/torch_optimizer.py
  2. 2
      ml-agents/mlagents/trainers/policy/torch_policy.py
  3. 44
      ml-agents/mlagents/trainers/ppo/optimizer_torch.py
  4. 125
      ml-agents/mlagents/trainers/sac/optimizer_torch.py
  5. 22
      ml-agents/mlagents/trainers/tests/torch/test_layers.py
  6. 17
      ml-agents/mlagents/trainers/tests/torch/test_networks.py
  7. 16
      ml-agents/mlagents/trainers/tests/torch/test_utils.py
  8. 72
      ml-agents/mlagents/trainers/torch/components/reward_providers/gail_reward_provider.py
  9. 36
      ml-agents/mlagents/trainers/torch/layers.py
  10. 46
      ml-agents/mlagents/trainers/torch/networks.py
  11. 10
      ml-agents/mlagents/trainers/torch/utils.py
  12. 1001
      ml-agents/mlagents/trainers/tests/torch/test.demo
  13. 150
      ml-agents/mlagents/trainers/tests/torch/test_bcmodule.py
  14. 446
      ml-agents/mlagents/trainers/tests/torch/testdcvis.demo
  15. 0
      ml-agents/mlagents/trainers/torch/components/bc/__init__.py
  16. 185
      ml-agents/mlagents/trainers/torch/components/bc/module.py

63
ml-agents/mlagents/trainers/optimizer/torch_optimizer.py


from typing import Dict, Optional, Tuple, List
import torch
import numpy as np
from mlagents_envs.base_env import DecisionSteps
from mlagents.trainers.components.bc.module import BCModule
from mlagents.trainers.trajectory import SplitObservations
from mlagents.trainers.torch.components.bc.module import BCModule
from mlagents.trainers.trajectory import SplitObservations
from mlagents.trainers.torch.utils import ModelUtils

self.global_step = torch.tensor(0)
self.bc_module: Optional[BCModule] = None
self.create_reward_signals(trainer_settings.reward_signals)
if trainer_settings.behavioral_cloning is not None:
self.bc_module = BCModule(
self.policy,
trainer_settings.behavioral_cloning,
policy_learning_rate=trainer_settings.hyperparameters.learning_rate,
default_batch_size=trainer_settings.hyperparameters.batch_size,
default_num_epoch=3,
)
def update(self, batch: AgentBuffer, num_sequences: int) -> Dict[str, float]:
pass

reward_signal, self.policy.behavior_spec, settings
)
def get_value_estimates(
self, decision_requests: DecisionSteps, idx: int, done: bool
) -> Dict[str, float]:
"""
Generates value estimates for bootstrapping.
:param decision_requests:
:param idx: Index in BrainInfo of agent.
:param done: Whether or not this is the last element of the episode,
in which case the value estimate will be 0.
:return: The value estimate dictionary with key being the name of the reward signal
and the value the corresponding value estimate.
"""
vec_vis_obs = SplitObservations.from_observations(decision_requests.obs)
value_estimates = self.policy.actor_critic.critic_pass(
np.expand_dims(vec_vis_obs.vector_observations[idx], 0),
np.expand_dims(vec_vis_obs.visual_observations[idx], 0),
)
value_estimates = {k: float(v) for k, v in value_estimates.items()}
# If we're done, reassign all of the value estimates that need terminal states.
if done:
for k in value_estimates:
if not self.reward_signals[k].ignore_done:
value_estimates[k] = 0.0
return value_estimates
def get_trajectory_value_estimates(
self, batch: AgentBuffer, next_obs: List[np.ndarray], done: bool
) -> Tuple[Dict[str, np.ndarray], Dict[str, float]]:

else:
visual_obs = []
memory = torch.zeros([1, len(vector_obs[0]), self.policy.m_size])
memory = torch.zeros([1, 1, self.policy.m_size])
next_obs = np.concatenate(next_obs, axis=-1)
next_obs = [ModelUtils.list_to_tensor(next_obs).unsqueeze(0)]
next_memory = torch.zeros([1, 1, self.policy.m_size])
vec_vis_obs = SplitObservations.from_observations(next_obs)
next_vec_obs = [
ModelUtils.list_to_tensor(vec_vis_obs.vector_observations).unsqueeze(0)
]
next_vis_obs = [
ModelUtils.list_to_tensor(_vis_ob).unsqueeze(0)
for _vis_ob in vec_vis_obs.visual_observations
]
value_estimates = self.policy.actor_critic.critic_pass(
vector_obs, visual_obs, memory
value_estimates, next_memory = self.policy.actor_critic.critic_pass(
vector_obs, visual_obs, memory, sequence_length=batch.num_experiences
next_value_estimate = self.policy.actor_critic.critic_pass(
next_obs, next_obs, next_memory
next_value_estimate, _ = self.policy.actor_critic.critic_pass(
next_vec_obs, next_vis_obs, next_memory, sequence_length=1
)
for name, estimate in value_estimates.items():

2
ml-agents/mlagents/trainers/policy/torch_policy.py


run_out["value"] = np.mean(list(run_out["value_heads"].values()), 0)
run_out["learning_rate"] = 0.0
if self.use_recurrent:
run_out["memories"] = memories.detach().cpu().numpy()
run_out["memory_out"] = memories.detach().cpu().numpy().squeeze(0)
return run_out
def get_action(

44
ml-agents/mlagents/trainers/ppo/optimizer_torch.py


old_values: Dict[str, torch.Tensor],
returns: Dict[str, torch.Tensor],
epsilon: float,
loss_masks: torch.Tensor,
Creates training-specific Tensorflow ops for PPO models.
:param returns:
:param old_values:
:param values:
Evaluates value loss for PPO.
:param values: Value output of the current network.
:param old_values: Value stored with experiences in buffer.
:param returns: Computed returns.
:param epsilon: Clipping value for value estimate.
:param loss_mask: Mask for losses. Used with LSTM to ignore 0'ed out experiences.
"""
value_losses = []
for name, head in values.items():

)
v_opt_a = (returns_tensor - head) ** 2
v_opt_b = (returns_tensor - clipped_value_estimate) ** 2
value_loss = torch.mean(torch.max(v_opt_a, v_opt_b))
value_loss = ModelUtils.masked_mean(torch.max(v_opt_a, v_opt_b), loss_masks)
def ppo_policy_loss(self, advantages, log_probs, old_log_probs, masks):
def ppo_policy_loss(
self,
advantages: torch.Tensor,
log_probs: torch.Tensor,
old_log_probs: torch.Tensor,
loss_masks: torch.Tensor,
) -> torch.Tensor:
Creates training-specific Tensorflow ops for PPO models.
:param masks:
:param advantages:
Evaluate PPO policy loss.
:param advantages: Computed advantages.
:param loss_masks: Mask for losses. Used with LSTM to ignore 0'ed out experiences.
"""
advantage = advantages.unsqueeze(-1)

p_opt_b = (
torch.clamp(r_theta, 1.0 - decay_epsilon, 1.0 + decay_epsilon) * advantage
)
policy_loss = -torch.mean(torch.min(p_opt_a, p_opt_b))
policy_loss = -1 * ModelUtils.masked_mean(
torch.min(p_opt_a, p_opt_b).flatten(), loss_masks
)
return policy_loss
@timed

memories=memories,
seq_len=self.policy.sequence_length,
)
value_loss = self.ppo_value_loss(values, old_values, returns, decay_eps)
loss_masks = ModelUtils.list_to_tensor(batch["masks"], dtype=torch.bool)
value_loss = self.ppo_value_loss(
values, old_values, returns, decay_eps, loss_masks
)
ModelUtils.list_to_tensor(batch["masks"], dtype=torch.int32),
loss_masks,
)
loss = (
policy_loss
+ 0.5 * value_loss
- decay_bet * ModelUtils.masked_mean(entropy.flatten(), loss_masks)
loss = policy_loss + 0.5 * value_loss - decay_bet * torch.mean(entropy)
# Set optimizer learning rate
ModelUtils.update_learning_rate(self.optimizer, decay_lr)

125
ml-agents/mlagents/trainers/sac/optimizer_torch.py


import numpy as np
from typing import Dict, List, Mapping, cast, Tuple
from typing import Dict, List, Mapping, cast, Tuple, Optional
import attr
from mlagents_envs.logging_util import get_logger
from mlagents_envs.base_env import ActionType

self,
vec_inputs: List[torch.Tensor],
vis_inputs: List[torch.Tensor],
actions: torch.Tensor = None,
actions: Optional[torch.Tensor] = None,
memories: Optional[torch.Tensor] = None,
sequence_length: int = 1,
q1_out, _ = self.q1_network(vec_inputs, vis_inputs, actions=actions)
q2_out, _ = self.q2_network(vec_inputs, vis_inputs, actions=actions)
q1_out, _ = self.q1_network(
vec_inputs,
vis_inputs,
actions=actions,
memories=memories,
sequence_length=sequence_length,
)
q2_out, _ = self.q2_network(
vec_inputs,
vis_inputs,
actions=actions,
memories=memories,
sequence_length=sequence_length,
)
return q1_out, q2_out
def __init__(self, policy: TorchPolicy, trainer_params: TrainerSettings):

for name in self.stream_names
}
# Critics should have 1/2 of the memory of the policy
critic_memory = policy_network_settings.memory
if critic_memory is not None:
critic_memory = attr.evolve(
critic_memory, memory_size=critic_memory.memory_size // 2
)
value_network_settings = attr.evolve(
policy_network_settings, memory=critic_memory
)
policy_network_settings,
value_network_settings,
policy_network_settings,
value_network_settings,
)
self.soft_update(self.policy.actor_critic.critic, self.target_network, 1.0)

* self.gammas[i]
* target_values[name]
)
_q1_loss = 0.5 * torch.mean(
loss_masks * torch.nn.functional.mse_loss(q_backup, q1_stream)
_q1_loss = 0.5 * ModelUtils.masked_mean(
torch.nn.functional.mse_loss(q_backup, q1_stream), loss_masks
_q2_loss = 0.5 * torch.mean(
loss_masks * torch.nn.functional.mse_loss(q_backup, q2_stream)
_q2_loss = 0.5 * ModelUtils.masked_mean(
torch.nn.functional.mse_loss(q_backup, q2_stream), loss_masks
)
q1_losses.append(_q1_loss)

v_backup = min_policy_qs[name] - torch.sum(
_ent_coef * log_probs, dim=1
)
# print(log_probs, v_backup, _ent_coef, loss_masks)
value_loss = 0.5 * torch.mean(
loss_masks * torch.nn.functional.mse_loss(values[name], v_backup)
value_loss = 0.5 * ModelUtils.masked_mean(
torch.nn.functional.mse_loss(values[name], v_backup), loss_masks
)
value_losses.append(value_loss)
else:

v_backup = min_policy_qs[name] - torch.mean(
branched_ent_bonus, axis=0
)
value_loss = 0.5 * torch.mean(
loss_masks
* torch.nn.functional.mse_loss(values[name], v_backup.squeeze())
value_loss = 0.5 * ModelUtils.masked_mean(
torch.pan><span class="n">nn.functional.mse_loss(values[namen><span class="p">], v_backup.squeeze()),
loss_masks,
)
value_losses.append(value_loss)
value_loss = torch.mean(torch.stack(value_losses))

if not discrete:
mean_q1 = mean_q1.unsqueeze(1)
batch_policy_loss = torch.mean(_ent_coef * log_probs - mean_q1, dim=1)
policy_loss = torch.mean(loss_masks * batch_policy_loss)
policy_loss = ModelUtils.masked_mean(batch_policy_loss, loss_masks)
else:
action_probs = log_probs.exp()
branched_per_action_ent = ModelUtils.break_into_branches(

target_current_diff = torch.squeeze(
target_current_diff_branched, axis=2
)
entropy_loss = -torch.mean(
loss_masks
* torch.mean(self._log_ent_coef * target_current_diff, axis=1)
entropy_loss = -1 * ModelUtils.masked_mean(
torch.mean(self._log_ent_coef * target_current_diff, axis=1), loss_masks
)
return entropy_loss

else:
actions = ModelUtils.list_to_tensor(batch["actions"], dtype=torch.long)
memories = [
memories_list = [
if len(memories) > 0:
memories = torch.stack(memories).unsqueeze(0)
# LSTM shouldn't have sequence length <1, but stop it from going out of the index if true.
offset = 1 if self.policy.sequence_length > 1 else 0
next_memories_list = [
ModelUtils.list_to_tensor(
batch["memory"][i][self.policy.m_size // 2 :]
) # only pass value part of memory to target network
for i in range(offset, len(batch["memory"]), self.policy.sequence_length)
]
if len(memories_list) > 0:
memories = torch.stack(memories_list).unsqueeze(0)
next_memories = torch.stack(next_memories_list).unsqueeze(0)
else:
memories = None
next_memories = None
# Q network memories are 0'ed out, since we don't have them during inference.
q_memories = torch.zeros_like(next_memories)
vis_obs: List[torch.Tensor] = []
next_vis_obs: List[torch.Tensor] = []
if self.policy.use_vis_obs:

)
if self.policy.use_continuous_act:
squeezed_actions = actions.squeeze(-1)
q1p_out, q2p_out = self.value_network(vec_obs, vis_obs, sampled_actions)
q1_out, q2_out = self.value_network(vec_obs, vis_obs, squeezed_actions)
q1p_out, q2p_out = self.value_network(
vec_obs,
vis_obs,
sampled_actions,
memories=q_memories,
sequence_length=self.policy.sequence_length,
)
q1_out, q2_out = self.value_network(
vec_obs,
vis_obs,
squeezed_actions,
memories=q_memories,
sequence_length=self.policy.sequence_length,
)
q1p_out, q2p_out = self.value_network(vec_obs, vis_obs)
q1_out, q2_out = self.value_network(vec_obs, vis_obs)
q1p_out, q2p_out = self.value_network(
vec_obs,
vis_obs,
memories=q_memories,
sequence_length=self.policy.sequence_length,
)
q1_out, q2_out = self.value_network(
vec_obs,
vis_obs,
memories=q_memories,
sequence_length=self.policy.sequence_length,
)
target_values, _ = self.target_network(next_vec_obs, next_vis_obs)
masks = ModelUtils.list_to_tensor(batch["masks"], dtype=torch.int32)
target_values, _ = self.target_network(
next_vec_obs,
next_vis_obs,
memories=next_memories,
sequence_length=self.policy.sequence_length,
)
masks = ModelUtils.list_to_tensor(batch["masks"], dtype=torch.bool)
use_discrete = not self.policy.use_continuous_act
dones = ModelUtils.list_to_tensor(batch["done"])

22
ml-agents/mlagents/trainers/tests/torch/test_layers.py


import torch
from mlagents.trainers.torch.layers import Swish, linear_layer, Initialization
from mlagents.trainers.torch.layers import (
Swish,
linear_layer,
lstm_layer,
Initialization,
)
def test_swish():

)
assert torch.all(torch.eq(layer.weight.data, torch.zeros_like(layer.weight.data)))
assert torch.all(torch.eq(layer.bias.data, torch.zeros_like(layer.bias.data)))
def test_lstm_layer():
torch.manual_seed(0)
# Test zero for LSTM
layer = lstm_layer(
4, 4, kernel_init=Initialization.Zero, bias_init=Initialization.Zero
)
for name, param in layer.named_parameters():
if "weight" in name:
assert torch.all(torch.eq(param.data, torch.zeros_like(param.data)))
elif "bias" in name:
assert torch.all(
torch.eq(param.data[4:8], torch.ones_like(param.data[4:8]))
)

17
ml-agents/mlagents/trainers/tests/torch/test_networks.py


obs_size = 4
seq_len = 16
network_settings = NetworkSettings(
memory=NetworkSettings.MemorySettings(sequence_length=seq_len, memory_size=4)
memory=NetworkSettings.MemorySettings(sequence_length=seq_len, memory_size=12)
optimizer = torch.optim.Adam(networkbody.parameters(), lr=3e-3)
optimizer = torch.optim.Adam(networkbody.parameters(), lr=3e-4)
for _ in range(100):
encoded, _ = networkbody([sample_obs], [], memories=torch.ones(1, seq_len, 4))
for _ in range(200):
encoded, _ = networkbody([sample_obs], [], memories=torch.ones(1, seq_len, 12))
# Try to force output to 1
loss = torch.nn.functional.mse_loss(encoded, torch.ones(encoded.shape))
optimizer.zero_grad()

# memories isn't always set to None, the network should be able to
# deal with that.
# Test critic pass
value_out = actor.critic_pass([sample_obs], [], memories=memories)
value_out, memories_out = actor.critic_pass([sample_obs], [], memories=memories)
assert memories_out.shape == memories.shape
dists, value_out, _ = actor.get_dist_and_value([sample_obs], [], memories=memories)
dists, value_out, mem_out = actor.get_dist_and_value(
[sample_obs], [], memories=memories
)
if mem_out is not None:
assert mem_out.shape == memories.shape
for dist in dists:
assert isinstance(dist, GaussianDistInstance)
for stream in stream_names:

16
ml-agents/mlagents/trainers/tests/torch/test_utils.py


assert entropies.shape == (1, len(dist_list))
# Make sure the first action has high probability than the others.
assert log_probs.flatten()[0] > log_probs.flatten()[1]
def test_masked_mean():
test_input = torch.tensor([1, 2, 3, 4, 5])
masks = torch.ones_like(test_input).bool()
mean = ModelUtils.masked_mean(test_input, masks=masks)
assert mean == 3.0
masks = torch.tensor([False, False, True, True, True])
mean = ModelUtils.masked_mean(test_input, masks=masks)
assert mean == 4.0
# Make sure it works if all masks are off
masks = torch.tensor([False, False, False, False, False])
mean = ModelUtils.masked_mean(test_input, masks=masks)
assert mean == 0.0

72
ml-agents/mlagents/trainers/torch/components/reward_providers/gail_reward_provider.py


expert_batch = self._demo_buffer.sample_mini_batch(
mini_batch.num_experiences, 1
)
loss, policy_mean_estimate, expert_mean_estimate, kl_loss = self._discriminator_network.compute_loss(
loss, stats_dict = self._discriminator_network.compute_loss(
stats_dict = {
"Losses/GAIL Discriminator Loss": loss.detach().cpu().numpy(),
"Policy/GAIL Policy Estimate": policy_mean_estimate.detach().cpu().numpy(),
"Policy/GAIL Expert Estimate": expert_mean_estimate.detach().cpu().numpy(),
}
if self._discriminator_network.use_vail:
stats_dict["Policy/GAIL Beta"] = (
self._discriminator_network.beta.detach().cpu().numpy()
)
stats_dict["Losses/GAIL KL Loss"] = kl_loss.detach().cpu().numpy()
return stats_dict

def __init__(self, specs: BehaviorSpec, settings: GAILSettings) -> None:
super().__init__()
self._policy_specs = specs
self.use_vail = settings.use_vail
self._use_vail = settings.use_vail
self._settings = settings
state_encoder_settings = NetworkSettings(

estimator_input_size = settings.encoding_size
if settings.use_vail:
estimator_input_size = self.z_size
self.z_sigma = torch.nn.Parameter(
self._z_sigma = torch.nn.Parameter(
self.z_mu_layer = linear_layer(
self._z_mu_layer = linear_layer(
self.beta = torch.nn.Parameter(
self._beta = torch.nn.Parameter(
self.estimator = torch.nn.Sequential(
self._estimator = torch.nn.Sequential(
linear_layer(estimator_input_size, 1), torch.nn.Sigmoid()
)

hidden = self.encoder(encoder_input)
z_mu: Optional[torch.Tensor] = None
if self._settings.use_vail:
z_mu = self.z_mu_layer(hidden)
hidden = torch.normal(z_mu, self.z_sigma * use_vail_noise)
estimate = self.estimator(hidden)
z_mu = self._z_mu_layer(hidden)
hidden = torch.normal(z_mu, self._z_sigma * use_vail_noise)
estimate = self._estimator(hidden)
return estimate, z_mu
def compute_loss(

Given a policy mini_batch and an expert mini_batch, computes the loss of the discriminator.
"""
total_loss = torch.zeros(1)
stats_dict: Dict[str, np.ndarray] = {}
policy_estimate, policy_mu = self.compute_estimate(
policy_batch, use_vail_noise=True
)

loss = -(
torch.log(expert_estimate * (1 - self.EPSILON))
+ torch.log(1.0 - policy_estimate * (1 - self.EPSILON))
stats_dict["Policy/GAIL Policy Estimate"] = (
policy_estimate.mean().detach().cpu().numpy()
)
stats_dict["Policy/GAIL Expert Estimate"] = (
expert_estimate.mean().detach().cpu().numpy()
)
discriminator_loss = -(
torch.log(expert_estimate + self.EPSILON)
+ torch.log(1.0 - policy_estimate + self.EPSILON)
kl_loss: Optional[torch.Tensor] = None
stats_dict["Losses/GAIL Loss"] = discriminator_loss.detach().cpu().numpy()
total_loss += discriminator_loss
+ (self.z_sigma ** 2).log()
+ (self._z_sigma ** 2).log()
- (self.z_sigma ** 2),
- (self._z_sigma ** 2),
vail_loss = self.beta * (kl_loss - self.mutual_information)
vail_loss = self._beta * (kl_loss - self.mutual_information)
self.beta.data = torch.max(
self.beta + self.alpha * (kl_loss - self.mutual_information),
self._beta.data = torch.max(
self._beta + self.alpha * (kl_loss - self.mutual_information),
loss += vail_loss
total_loss += vail_loss
stats_dict["Policy/GAIL Beta"] = self._beta.detach().cpu().numpy()
stats_dict["Losses/GAIL KL Loss"] = kl_loss.detach().cpu().numpy()
loss += self.gradient_penalty_weight * self.compute_gradient_magnitude(
policy_batch, expert_batch
total_loss += (
self.gradient_penalty_weight
* self.compute_gradient_magnitude(policy_batch, expert_batch)
return loss, torch.mean(policy_estimate), torch.mean(expert_estimate), kl_loss
return total_loss, stats_dict
def compute_gradient_magnitude(
self, policy_batch: AgentBuffer, expert_batch: AgentBuffer

hidden = self.encoder(encoder_input)
if self._settings.use_vail:
use_vail_noise = True
z_mu = self.z_mu_layer(hidden)
hidden = torch.normal(z_mu, self.z_sigma * use_vail_noise)
hidden = self.estimator(hidden)
z_mu = self._z_mu_layer(hidden)
hidden = torch.normal(z_mu, self._z_sigma * use_vail_noise)
hidden = self._estimator(hidden)
estimate = torch.mean(torch.sum(hidden, dim=1))
gradient = torch.autograd.grad(estimate, encoder_input)[0]
# Norm's gradient could be NaN at 0. Use our own safe_norm

36
ml-agents/mlagents/trainers/torch/layers.py


layer.weight.data *= kernel_gain
_init_methods[bias_init](layer.bias.data)
return layer
def lstm_layer(
input_size: int,
hidden_size: int,
num_layers: int = 1,
batch_first: bool = True,
forget_bias: float = 1.0,
kernel_init: Initialization = Initialization.XavierGlorotUniform,
bias_init: Initialization = Initialization.Zero,
) -> torch.nn.Module:
"""
Creates a torch.nn.LSTM and initializes its weights and biases. Provides a
forget_bias offset like is done in TensorFlow.
"""
lstm = torch.nn.LSTM(input_size, hidden_size, num_layers, batch_first=batch_first)
# Add forget_bias to forget gate bias
for name, param in lstm.named_parameters():
# Each weight and bias is a concatenation of 4 matrices
if "weight" in name:
for idx in range(4):
block_size = param.shape[0] // 4
_init_methods[kernel_init](
param.data[idx * block_size : (idx + 1) * block_size]
)
if "bias" in name:
for idx in range(4):
block_size = param.shape[0] // 4
_init_methods[bias_init](
param.data[idx * block_size : (idx + 1) * block_size]
)
if idx == 1:
param.data[idx * block_size : (idx + 1) * block_size].add_(
forget_bias
)
return lstm

46
ml-agents/mlagents/trainers/torch/networks.py


from mlagents.trainers.settings import NetworkSettings
from mlagents.trainers.torch.utils import ModelUtils
from mlagents.trainers.torch.decoders import ValueHeads
from mlagents.trainers.torch.layers import lstm_layer
ActivationFunction = Callable[[torch.Tensor], torch.Tensor]
EncoderFunction = Callable[

)
if self.use_lstm:
self.lstm = nn.LSTM(self.h_size, self.m_size // 2, 1)
self.lstm = lstm_layer(self.h_size, self.m_size // 2, batch_first=True)
else:
self.lstm = None

encoding += _enc
if self.use_lstm:
encoding = encoding.view([sequence_length, -1, self.h_size])
# Resize to (batch, sequence length, encoding size)
encoding = encoding.reshape([-1, sequence_length, self.h_size])
encoding, memories = self.lstm(
encoding.contiguous(),
(memories[0].contiguous(), memories[1].contiguous()),
)
encoding = encoding.view([-1, self.m_size // 2])
encoding, memories = self.lstm(encoding, memories)
encoding = encoding.reshape([-1, self.m_size // 2])
memories = torch.cat(memories, dim=-1)
return encoding, memories

vec_inputs: List[torch.Tensor],
vis_inputs: List[torch.Tensor],
memories: Optional[torch.Tensor] = None,
) -> Dict[str, torch.Tensor]:
sequence_length: int = 1,
) -> Tuple[Dict[str, torch.Tensor], torch.Tensor]:
"""
Get value outputs for the given obs.
:param vec_inputs: List of vector inputs as tensors.

vec_inputs: List[torch.Tensor],
vis_inputs: List[torch.Tensor],
memories: Optional[torch.Tensor] = None,
) -> Dict[str, torch.Tensor]:
encoding, _ = self.network_body(vec_inputs, vis_inputs, memories=memories)
return self.value_heads(encoding)
sequence_length: int = 1,
) -> Tuple[Dict[str, torch.Tensor], torch.Tensor]:
encoding, memories_out = self.network_body(
vec_inputs, vis_inputs, memories=memories, sequence_length=sequence_length
)
return self.value_heads(encoding), memories_out
def get_dist_and_value(
self,

vec_inputs: List[torch.Tensor],
vis_inputs: List[torch.Tensor],
memories: Optional[torch.Tensor] = None,
) -> Dict[str, torch.Tensor]:
sequence_length: int = 1,
) -> Tuple[Dict[str, torch.Tensor], torch.Tensor]:
actor_mem, critic_mem = None, None
_, critic_mem = torch.split(memories, self.half_mem_size, -1)
actor_mem, critic_mem = torch.split(memories, self.half_mem_size, -1)
value_outputs, critic_mem_out = self.critic(
vec_inputs, vis_inputs, memories=critic_mem, sequence_length=sequence_length
)
if actor_mem is not None:
# Make memories with the actor mem unchanged
memories_out = torch.cat([actor_mem, critic_mem_out], dim=-1)
critic_mem = None
value_outputs, _memories = self.critic(
vec_inputs, vis_inputs, memories=critic_mem
)
return value_outputs
memories_out = None
return value_outputs, memories_out
def get_dist_and_value(
self,

vec_inputs, vis_inputs, memories=critic_mem, sequence_length=sequence_length
)
if self.use_lstm:
mem_out = torch.cat([actor_mem_outs, critic_mem_outs], dim=1)
mem_out = torch.cat([actor_mem_outs, critic_mem_outs], dim=-1)
else:
mem_out = None
return dists, value_outputs, mem_out

10
ml-agents/mlagents/trainers/torch/utils.py


else:
all_probs = torch.cat(all_probs_list, dim=-1)
return log_probs, entropies, all_probs
@staticmethod
def masked_mean(tensor: torch.Tensor, masks: torch.Tensor) -> torch.Tensor:
"""
Returns the mean of the tensor but ignoring the values specified by masks.
Used for masking out loss functions.
:param tensor: Tensor which needs mean computation.
:param masks: Boolean tensor of masks with same dimension as tensor.
"""
return (tensor * masks).sum() / torch.clamp(masks.float().sum(), min=1.0)

1001
ml-agents/mlagents/trainers/tests/torch/test.demo
文件差异内容过多而无法显示
查看文件

150
ml-agents/mlagents/trainers/tests/torch/test_bcmodule.py


from unittest.mock import MagicMock
import pytest
import mlagents.trainers.tests.mock_brain as mb
import numpy as np
import os
from mlagents.trainers.policy.torch_policy import TorchPolicy
from mlagents.trainers.torch.components.bc.module import BCModule
from mlagents.trainers.settings import (
TrainerSettings,
BehavioralCloningSettings,
NetworkSettings,
)
def create_bc_module(mock_behavior_specs, bc_settings, use_rnn, tanhresample):
# model_path = env.external_brain_names[0]
trainer_config = TrainerSettings()
trainer_config.network_settings.memory = (
NetworkSettings.MemorySettings() if use_rnn else None
)
policy = TorchPolicy(
0,
mock_behavior_specs,
trainer_config,
"test",
False,
tanhresample,
tanhresample,
)
bc_module = BCModule(
policy,
settings=bc_settings,
policy_learning_rate=trainer_config.hyperparameters.learning_rate,
default_batch_size=trainer_config.hyperparameters.batch_size,
default_num_epoch=3,
)
return bc_module
# Test default values
def test_bcmodule_defaults():
# See if default values match
mock_specs = mb.create_mock_3dball_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo"
)
bc_module = create_bc_module(mock_specs, bc_settings, False, False)
assert bc_module.num_epoch == 3
assert bc_module.batch_size == TrainerSettings().hyperparameters.batch_size
# Assign strange values and see if it overrides properly
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo",
num_epoch=100,
batch_size=10000,
)
bc_module = create_bc_module(mock_specs, bc_settings, False, False)
assert bc_module.num_epoch == 100
assert bc_module.batch_size == 10000
# Test with continuous control env and vector actions
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_update(is_sac):
mock_specs = mb.create_mock_3dball_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo"
)
bc_module = create_bc_module(mock_specs, bc_settings, False, is_sac)
stats = bc_module.update()
for _, item in stats.items():
assert isinstance(item, np.float32)
# Test with constant pretraining learning rate
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_constant_lr_update(is_sac):
mock_specs = mb.create_mock_3dball_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo",
steps=0,
)
bc_module = create_bc_module(mock_specs, bc_settings, False, is_sac)
stats = bc_module.update()
for _, item in stats.items():
assert isinstance(item, np.float32)
old_learning_rate = bc_module.current_lr
_ = bc_module.update()
assert old_learning_rate == bc_module.current_lr
# Test with constant pretraining learning rate
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_linear_lr_update(is_sac):
mock_specs = mb.create_mock_3dball_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo",
steps=100,
)
bc_module = create_bc_module(mock_specs, bc_settings, False, is_sac)
# Should decay by 10/100 * 0.0003 = 0.00003
bc_module.policy.get_current_step = MagicMock(return_value=10)
old_learning_rate = bc_module.current_lr
_ = bc_module.update()
assert old_learning_rate - 0.00003 == pytest.approx(bc_module.current_lr, abs=0.01)
# Test with RNN
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_rnn_update(is_sac):
mock_specs = mb.create_mock_3dball_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "test.demo"
)
bc_module = create_bc_module(mock_specs, bc_settings, True, is_sac)
stats = bc_module.update()
for _, item in stats.items():
assert isinstance(item, np.float32)
# Test with discrete control and visual observations
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_dc_visual_update(is_sac):
mock_specs = mb.create_mock_banana_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "testdcvis.demo"
)
bc_module = create_bc_module(mock_specs, bc_settings, False, is_sac)
stats = bc_module.update()
for _, item in stats.items():
assert isinstance(item, np.float32)
# Test with discrete control, visual observations and RNN
@pytest.mark.parametrize("is_sac", [True, False], ids=["sac", "ppo"])
def test_bcmodule_rnn_dc_update(is_sac):
mock_specs = mb.create_mock_banana_behavior_specs()
bc_settings = BehavioralCloningSettings(
demo_path=os.path.dirname(os.path.abspath(__file__)) + "/" + "testdcvis.demo"
)
bc_module = create_bc_module(mock_specs, bc_settings, True, is_sac)
stats = bc_module.update()
for _, item in stats.items():
assert isinstance(item, np.float32)
if __name__ == "__main__":
pytest.main()

446
ml-agents/mlagents/trainers/tests/torch/testdcvis.demo


bcvis& -****:VisualFoodCollectorLearning�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�ndE� �D,������� D$|!19?EF�--�@0^��ڡ�oU�z�}�glK��v��8�8�}�ww��?�,�yg�ĉw#���o;�v~�h�~���)��o;�� �c��Mo^;���/p��/p��mߎ};� �@;� ��v۷cߎ�'Ў�7����طc�� ,�د.//G������<:`��U_on�T]��W#'�W���׉�ÎN޵rqq�r��S��:������ms��:9 ���-o��y�䅑W̟�������g~�9��f�?>�������׷��?�?������ֱ.�/�S��v��\]_'K���W���z�������:�!�R�&�M~�>K�ۿ�<H �<www �$��wIډos��\o����3\��N �f�>��yr��ӈG �0��;;���UX���,R8 I�H$G����<�g?9(K�}V�T �K�ʘ�z �˗_eL��n����@A�`���`����#������V5:)��͌EK ������((%�I��7����NNH���[��LB��,��f���@�0D��ّ��%�M�NU����G�~jP�������t�I��!#����Iç��ؗ>� )��T�F4�Y��1Ĥ!%���ن ��0��9� �� $�2�d"���<K�����11ϼc��yQR��Q�T��Z�@� �Vb�]"�u�FN�j7�#�P�(\���&q�D��h+���m%����%�ȪS���%�a����Q��L�EI �$�(�8�`�^�D[dE?�6�ʧ�6A��R)bI��1��{I�+��*�!��E� ���h��-=_xH]��G0\�3�V�f.�cI��gp����3�]���
F�|���d���� �x�����(�{F��[ AL����|�hv��<���]V4�^���pAH�W�H"d��G����P�uż���A�A$��8�O��;��Ó��ý�UqpI^�(P�g�|���{��?G@[1��M�9h�E���샔�6���b��j�m-h�ڡ��"q"&
b1�,)��p'�����s
͗b�$P ���� IԳN��O0s��J�)�Q����=�bs`��s)&4|H�@ujٸ��6�0�䃞�{\x�t8����6��I��Wꝏ�����c��k�k.���؈�Kh��������rD�^��fRݐ��.��BG�/<'�:C�Yt��S�&b�]%תz�Ǿ�R� �����}��� ��¿�&�>���t��uel#��w���T����j�����K�X��Ô���r��.#?�wqMD��D ��r�`(�N���'�; m��ko� ����c���I�R�i�Э� /��6��d�`�9�j��D�̢O-�����7�S�Qv]U�iD� ���|5ÕI�IEND�B`�j
"
�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�ndE� �D,������� D$|!19?EF�--�@0^��ڡ�oU�z�}�glK��v��8�8�}�ww��?�,�yg�ĉw#���o;�v~�h�~���)��o;�� �c��Mo^;���/p��/p��mߎ};� �@;� ��v۷cߎ�'Ў�7����طc�� ,�د.//G������<:`��U_on�T]��W#'�W���׉�ÎN޵rqq�r��S��:������ms��:9 ���-o��y�䅑W̟�������g~�9��f�?>�������׷��?�?������ֱ.�/�S��v��\]_'K���W���z�������:�!�R�&�M~�>K�ۿ�<H �<www �$��wIډos��\o����3\��N �f�>��yr��ӈG �0��;;���UX���,R8 I�H$G����<�g?9(K�}V�T �K�ʘ�z �˗_eL��n����@A�`���`����#������V5:)��͌EK ������((%�I��7����NNH���[��LB��,��f���@�0D��ّ��%�M�NU����G�~jP�������t�I��!#����Iç��ؗ>� )��T�F4�Y��1Ĥ!%���ن ��0��9� �� $�2�d"���<K�����11ϼc��yQR��Q�T��Z�@� �Vb�]"�u�FN�j7�#�P�(\���&q�D��h+���m%����%�ȪS���%�a����Q��L�EI �$�(�8�`�^�D[dE?�6�ʧ�6A��R)bI��1��{I�+��*�!��E� ���h��-=_xH]��G0\�3�V�f.�cI��gp����3�]���
F�|���d���� �x�����(�{F��[ AL����|�hv��<���]V4�^���pAH�W�H"d��G����P�uż���A�A$��8�O��;��Ó��ý�UqpI^�(P�g�|���{��?G@[1��M�9h�E���샔�6���b��j�m-h�ڡ��"q"&
b1�,)��p'�����s
͗b�$P ���� IԳN��O0s��J�)�Q����=�bs`��s)&4|H�@ujٸ��6�0�䃞�{\x�t8����6��I��Wꝏ�����c��k�k.���؈�Kh��������rD�^��fRݐ��.��BG�/<'�:C�Yt��S�&b�]%תz�Ǿ�R� �����}��� ��¿�&�>���t��uel#��w���T����j�����K�X��Ô���r��.#?�wqMD��D ��r�`(�N���'�; m��ko� ����c���I�R�i�Э� /��6��d�`�9�j��D�̢O-�����7�S�Qv]U�iD� ���|5ÕI�IEND�B`�j
"
�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�ndE� �D,������� D$|!19?EF�--�@0^��ڡ�oU�z�}�glK��v��8�8�}�ww��?�,�yg�ĉw#���o;�v~�h�~���)��o;�� �c��Mo^;���/p��/p��mߎ};� �@;� ��v۷cߎ�'Ў�7����طc�� ,�د.//G������<:`��U_on�T]��W#'�W���׉�ÎN޵rqq�r��S��:������ms��:9 ���-o��y�䅑W̟�������g~�9��f�?>�������׷��?�?������ֱ.�/�S��v��\]_'K���W���z�������:�!�R�&�M~�>K�ۿ�<H �<www �$��wIډos��\o����3\��N �f�>��yr��ӈG �0��;;���UX���,R8 I�H$G����<�g?9(K�}V�T �K�ʘ�z �˗_eL��n����@A�`���`����#������V5:)��͌EK ������((%�I��7����NNH���[��LB��,��f���@�0D��ّ��%�M�NU����G�~jP�������t�I��!#����Iç��ؗ>� )��T�F4�Y��1Ĥ!%���ن ��0��9� �� $�2�d"���<K�����11ϼc��yQR��Q�T��Z�@� �Vb�]"�u�FN�j7�#�P�(\���&q�D��h+���m%����%�ȪS���%�a����Q��L�EI �$�(�8�`�^�D[dE?�6�ʧ�6A��R)bI��1��{I�+��*�!��E� ���h��-=_xH]��G0\�3�V�f.�cI��gp����3�]���
F�|���d���� �x�����(�{F��[ AL����|�hv��<���]V4�^���pAH�W�H"d��G����P�uż���A�A$��8�O��;��Ó��ý�UqpI^�(P�g�|���{��?G@[1��M�9h�E���샔�6���b��j�m-h�ڡ��"q"&
b1�,)��p'�����s
͗b�$P ���� IԳN��O0s��J�)�Q����=�bs`��s)&4|H�@ujٸ��6�0�䃞�{\x�t8����6��I��Wꝏ�����c��k�k.���؈�Kh��������rD�^��fRݐ��.��BG�/<'�:C�Yt��S�&b�]%תz�Ǿ�R� �����}��� ��¿�&�>���t��uel#��w���T����j�����K�X��Ô���r��.#?�wqMD��D ��r�`(�N���'�; m��ko� ����c���I�R�i�Э� /��6��d�`�9�j��D�̢O-�����7�S�Qv]U�iD� ���|5ÕI�IEND�B`�j
"
�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�ndE� �D,������� D$|!19?EF�--�@0^��ڡ�oU�z�}�glK��v��8�8�}�ww��?�,�yg�ĉw#���o;�v~�h�~���)��o;�� �c��Mo^;���/p��/p��mߎ};� �@;� ��v۷cߎ�'Ў�7����طc�� ,�د.//G������<:`��U_on�T]��W#'�W���׉�ÎN޵rqq�r��S��:������ms��:9 ���-o��y�䅑W̟�������g~�9��f�?>�������׷��?�?������ֱ.�/�S��v��\]_'K���W���z�������:�!�R�&�M~�>K�ۿ�<H �<www �$��wIډos��\o����3\��N �f�>��yr��ӈG �0��;;���UX���,R8 I�H$G����<�g?9(K�}V�T �K�ʘ�z �˗_eL��n����@A�`���`����#������V5:)��͌EK ������((%�I��7����NNH���[��LB��,��f���@�0D��ّ��%�M�NU����G�~jP�������t�I��!#����Iç��ؗ>� )��T�F4�Y��1Ĥ!%���ن ��0��9� �� $�2�d"���<K�����11ϼc��yQR��Q�T��Z�@� �Vb�]"�u�FN�j7�#�P�(\���&q�D��h+���m%����%�ȪS���%�a����Q��L�EI �$�(�8�`�^�D[dE?�6�ʧ�6A��R)bI��1��{I�+��*�!��E� ���h��-=_xH]��G0\�3�V�f.�cI��gp����3�]���
F�|���d���� �x�����(�{F��[ AL����|�hv��<���]V4�^���pAH�W�H"d��G����P�uż���A�A$��8�O��;��Ó��ý�UqpI^�(P�g�|���{��?G@[1��M�9h�E���샔�6���b��j�m-h�ڡ��"q"&
b1�,)��p'�����s
͗b�$P ���� IԳN��O0s��J�)�Q����=�bs`��s)&4|H�@ujٸ��6�0�䃞�{\x�t8����6��I��Wꝏ�����c��k�k.���؈�Kh��������rD�^��fRݐ��.��BG�/<'�:C�Yt��S�&b�]%תz�Ǿ�R� �����}��� ��¿�&�>���t��uel#��w���T����j�����K�X��Ô���r��.#?�wqMD��D ��r�`(�N���'�; m��ko� ����c���I�R�i�Э� /��6��d�`�9�j��D�̢O-�����7�S�Qv]U�iD� ���|5ÕI�IEND�B`�j
"
�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�ndE� �D,������� D$|!19?EF�--�@0^��ڡ�oU�z�}�glK��v��8�8�}�ww��?�,�yg�ĉw#���o;�v~�h�~���)��o;�� �c��Mo^;���/p��/p��mߎ};� �@;� ��v۷cߎ�'Ў�7����طc�� ,�د.//G������<:`��U_on�T]��W#'�W���׉�ÎN޵rqq�r��S��:������ms��:9 ���-o��y�䅑W̟�������g~�9��f�?>�������׷��?�?������ֱ.�/�S��v��\]_'K���W���z�������:�!�R�&�M~�>K�ۿ�<H �<www �$��wIډos��\o����3\��N �f�>��yr��ӈG �0��;;���UX���,R8 I�H$G����<�g?9(K�}V�T �K�ʘ�z �˗_eL��n����@A�`���`����#������V5:)��͌EK ������((%�I��7����NNH���[��LB��,��f���@�0D��ّ��%�M�NU����G�~jP�������t�I��!#����Iç��ؗ>� )��T�F4�Y��1Ĥ!%���ن ��0��9� �� $�2�d"���<K�����11ϼc��yQR��Q�T��Z�@� �Vb�]"�u�FN�j7�#�P�(\���&q�D��h+���m%����%�ȪS���%�a����Q��L�EI �$�(�8�`�^�D[dE?�6�ʧ�6A��R)bI��1��{I�+��*�!��E� ���h��-=_xH]��G0\�3�V�f.�cI��gp����3�]���
F�|���d���� �x�����(�{F��[ AL����|�hv��<���]V4�^���pAH�W�H"d��G����P�uż���A�A$��8�O��;��Ó��ý�UqpI^�(P�g�|���{��?G@[1��M�9h�E���샔�6���b��j�m-h�ڡ��"q"&
b1�,)��p'�����s
͗b�$P ���� IԳN��O0s��J�)�Q����=�bs`��s)&4|H�@ujٸ��6�0�䃞�{\x�t8����6��I��Wꝏ�����c��k�k.���؈�Kh��������rD�^��fRݐ��.��BG�/<'�:C�Yt��S�&b�]%תz�Ǿ�R� �����}��� ��¿�&�>���t��uel#��w���T����j�����K�X��Ô���r��.#?�wqMD��D ��r�`(�N���'�; m��ko� ����c���I�R�i�Э� /��6��d�`�9�j��D�̢O-�����7�S�Qv]U�iD� ���|5ÕI�IEND�B`�j
"
@�?�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Xͮ�D����$W�@�-j T]!$6���'�Ux ��a�W@����X���R����$�mn�1ߙ��s��6�l�u�����o��ط�7��4���J�x�䛺��ʷ+��h۾���(�+߮|g�m�.z{��m߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g��m׾���������q�������F�&Z����>o@���9�n����|����|���������ۏ��8�����׵ ���򗿿�<K����x����P����`����1��݋4�����+��6��G7�E�H���Y2;N�w������OGx�&i��=��f~��ܣ8�:�� ��{!�:Nn��?=� �/� �� ���?>�%�EP���A�q7~v�?�͵�����X��ߣG��R*�*���KB�/��.]r|�x���ePA� ��"_fK�1X,�%�A�[�'y��7��y��]F��Mje"�%@���lET����X��#r���E�<��<���i��sh�<�=|��wG�9��G�b=<�آ��[j[�|*��ّY��^ ��B�"q¡�����q6���ϔ?ܘ�G�����8 �3 �BhORn���1�̕=���P0q���a ��Ģ#����/��Φ'��ω�"�γ�Lܳ ""Po�`0@�q���+Ec���d�<��\�r����Y��^9�|�o �������hoZd�g0�fx��NOSO��T��0����"v��Y:�CSL]�Y({��ɘ3��W�V��^�����AX>�@Ap6:û ˚/��p4<�@0?Mg�Y��7,����!�A ֟ձb�% Jn�yi�_Q�5��Ѻ*IM���)�C
/O^�8ϋe^�t���s���<z�p��0�H� �l��dX�
V :W�\��y�Q��7��>9Ӱ �=�����>�n��n�t"�ǿ(���{�˃��Ӵ��FH]��Wް����N�a���liݸ��w ��?z~��"���Ǚ7?��$]�K��"�謿��Xvn���I����2׵�Q޹�%s�Hc�ge�J�7���#��h~��� S8��H�s�_�� ܨ������VF�OQ��}n�?��<�L��ݧ��a���X�0�=@������bBi���˃��v}��Hd��W%dV�/F�+�1f9�§�gZ��$�����D�ҹJ�����̝���N�dȡVd!h5bo�%K��H~�������A�o�s"�7���Hi��(���i�<
�( VHLS�m�)��5.�&��K�_��i��,��� ����,������U�d�_���=I&����'��R`u�+�Z�+�s�V�Uh ��m��ל5��b�[����g��j�0ب����8�
��\��F����B��2����a��c��� ��5o��݄�3ʓI'e~�I�9VL���8��E�'\�=v�Q�$mﰁu�u�}����&ֵ��0��J��Y8&`�Y���z���<���NF0 Yg����JՙJ�\��]��B
������ ������s���6q�D���Z�i>�[�D0<���E���럞N�z��H2b-�3�?�J �2s�5?v����z�=/���1�P;��D;�.UE��l��1#����#)�v{�M�ʫi@�yrH��^��d ����`�_ψF��[���R{6�~ɛT)E�J�~��\��(�^u����몔�J��ެ��u=b/|y>����˛h� s�D�Ԩـ���G �Ul��Ni���I��@�Ƨ����~k�&Vx��5���G^��^�"ۃ����y�:a5E�dmF2K��M����Ug���-6�uI�z4K��t�F鉘��Ȗ��dv�^��-��-�>��b.�>��`V� �������jR�r<s���ω�D`V��q�Y�ʨ���*�Ì�N#ov�Y
���tT���i���{�~������ޔ�:x�P�a栭��������i��_IEND�B`�j
"
@�?�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Xͮ�D����$W�@�-j T]!$6���'�Ux ��a�W@����X���R����$�mn�1ߙ��s��6�l�u�����o��ط�7��4���J�x�䛺��ʷ+��h۾���(�+߮|g�m�.z{��m߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g��m׾���������q�������F�&Z����>o@���9�n����|����|���������ۏ��8�����׵ ���򗿿�<K����x����P����`����1��݋4�����+��6��G7�E�H���Y2;N�w������OGx�&i��=��f~��ܣ8�:�� ��{!�:Nn��?=� �/� �� ���?>�%�EP���A�q7~v�?�͵�����X��ߣG��R*�*���KB�/��.]r|�x���ePA� ��"_fK�1X,�%�A�[�'y��7��y��]F��Mje"�%@���lET����X��#r���E�<��<���i��sh�<�=|��wG�9��G�b=<�آ��[j[�|*��ّY��^ ��B�"q¡�����q6���ϔ?ܘ�G�����8 �3 �BhORn���1�̕=���P0q���a ��Ģ#����/��Φ'��ω�"�γ�Lܳ ""Po�`0@�q���+Ec���d�<��\�r����Y��^9�|�o �������hoZd�g0�fx��NOSO��T��0����"v��Y:�CSL]�Y({��ɘ3��W�V��^�����AX>�@Ap6:û ˚/��p4<�@0?Mg�Y��7,����!�A ֟ձb�% Jn�yi�_Q�5��Ѻ*IM���)�C
/O^�8ϋe^�t���s���<z�p��0�H� �l��dX�
V :W�\��y�Q��7��>9Ӱ �=�����>�n��n�t"�ǿ(���{�˃��Ӵ��FH]��Wް����N�a���liݸ��w ��?z~��"���Ǚ7?��$]�K��"�謿��Xvn���I����2׵�Q޹�%s�Hc�ge�J�7���#��h~��� S8��H�s�_�� ܨ������VF�OQ��}n�?��<�L��ݧ��a���X�0�=@������bBi���˃��v}��Hd��W%dV�/F�+�1f9�§�gZ��$�����D�ҹJ�����̝���N�dȡVd!h5bo�%K��H~�������A�o�s"�7���Hi��(���i�<
�( VHLS�m�)��5.�&��K�_��i��,��� ����,������U�d�_���=I&����'��R`u�+�Z�+�s�V�Uh ��m��ל5��b�[����g��j�0ب����8�
��\��F����B��2����a��c��� ��5o��݄�3ʓI'e~�I�9VL���8��E�'\�=v�Q�$mﰁu�u�}����&ֵ��0��J��Y8&`�Y���z���<���NF0 Yg����JՙJ�\��]��B
������ ������s���6q�D���Z�i>�[�D0<���E���럞N�z��H2b-�3�?�J �2s�5?v����z�=/���1�P;��D;�.UE��l��1#����#)�v{�M�ʫi@�yrH��^��d ����`�_ψF��[���R{6�~ɛT)E�J�~��\��(�^u����몔�J��ެ��u=b/|y>����˛h� s�D�Ԩـ���G �Ul��Ni���I��@�Ƨ����~k�&Vx��5���G^��^�"ۃ����y�:a5E�dmF2K��M����Ug���-6�uI�z4K��t�F鉘��Ȗ��dv�^��-��-�>��b.�>��`V� �������jR�r<s���ω�D`V��q�Y�ʨ���*�Ì�N#ov�Y
���tT���i���{�~������ޔ�:x�P�a栭��������i��_IEND�B`�j
"
@�?�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Xͮ�D����$W�@�-j T]!$6���'�Ux ��a�W@����X���R����$�mn�1ߙ��s��6�l�u�����o��ط�7��4���J�x�䛺��ʷ+��h۾���(�+߮|g�m�.z{��m߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g��m׾���������q�������F�&Z����>o@���9�n����|����|���������ۏ��8�����׵ ���򗿿�<K����x����P����`����1��݋4�����+��6��G7�E�H���Y2;N�w������OGx�&i��=��f~��ܣ8�:�� ��{!�:Nn��?=� �/� �� ���?>�%�EP���A�q7~v�?�͵�����X��ߣG��R*�*���KB�/��.]r|�x���ePA� ��"_fK�1X,�%�A�[�'y��7��y��]F��Mje"�%@���lET����X��#r���E�<��<���i��sh�<�=|��wG�9��G�b=<�آ��[j[�|*��ّY��^ ��B�"q¡�����q6���ϔ?ܘ�G�����8 �3 �BhORn���1�̕=���P0q���a ��Ģ#����/��Φ'��ω�"�γ�Lܳ ""Po�`0@�q���+Ec���d�<��\�r����Y��^9�|�o �������hoZd�g0�fx��NOSO��T��0����"v��Y:�CSL]�Y({��ɘ3��W�V��^�����AX>�@Ap6:û ˚/��p4<�@0?Mg�Y��7,����!�A ֟ձb�% Jn�yi�_Q�5��Ѻ*IM���)�C
/O^�8ϋe^�t���s���<z�p��0�H� �l��dX�
V :W�\��y�Q��7��>9Ӱ �=�����>�n��n�t"�ǿ(���{�˃��Ӵ��FH]��Wް����N�a���liݸ��w ��?z~��"���Ǚ7?��$]�K��"�謿��Xvn���I����2׵�Q޹�%s�Hc�ge�J�7���#��h~��� S8��H�s�_�� ܨ������VF�OQ��}n�?��<�L��ݧ��a���X�0�=@������bBi���˃��v}��Hd��W%dV�/F�+�1f9�§�gZ��$�����D�ҹJ�����̝���N�dȡVd!h5bo�%K��H~�������A�o�s"�7���Hi��(���i�<
�( VHLS�m�)��5.�&��K�_��i��,��� ����,������U�d�_���=I&����'��R`u�+�Z�+�s�V�Uh ��m��ל5��b�[����g��j�0ب����8�
��\��F����B��2����a��c��� ��5o��݄�3ʓI'e~�I�9VL���8��E�'\�=v�Q�$mﰁu�u�}����&ֵ��0��J��Y8&`�Y���z���<���NF0 Yg����JՙJ�\��]��B
������ ������s���6q�D���Z�i>�[�D0<���E���럞N�z��H2b-�3�?�J �2s�5?v����z�=/���1�P;��D;�.UE��l��1#����#)�v{�M�ʫi@�yrH��^��d ����`�_ψF��[���R{6�~ɛT)E�J�~��\��(�^u����몔�J��ެ��u=b/|y>����˛h� s�D�Ԩـ���G �Ul��Ni���I��@�Ƨ����~k�&Vx��5���G^��^�"ۃ����y�:a5E�dmF2K��M����Ug���-6�uI�z4K��t�F鉘��Ȗ��dv�^��-��-�>��b.�>��`V� �������jR�r<s���ω�D`V��q�Y�ʨ���*�Ì�N#ov�Y
���tT���i���{�~������ޔ�:x�P�a栭��������i��_IEND�B`�j
"
@�?�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Xͮ�D����$W�@�-j T]!$6���'�Ux ��a�W@����X���R����$�mn�1ߙ��s��6�l�u�����o��ط�7��4���J�x�䛺��ʷ+��h۾���(�+߮|g�m�.z{��m߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g��m׾���������q�������F�&Z����>o@���9�n����|����|���������ۏ��8�����׵ ���򗿿�<K����x����P����`����1��݋4�����+��6��G7�E�H���Y2;N�w������OGx�&i��=��f~��ܣ8�:�� ��{!�:Nn��?=� �/� �� ���?>�%�EP���A�q7~v�?�͵�����X��ߣG��R*�*���KB�/��.]r|�x���ePA� ��"_fK�1X,�%�A�[�'y��7��y��]F��Mje"�%@���lET����X��#r���E�<��<���i��sh�<�=|��wG�9��G�b=<�آ��[j[�|*��ّY��^ ��B�"q¡�����q6���ϔ?ܘ�G�����8 �3 �BhORn���1�̕=���P0q���a ��Ģ#����/��Φ'��ω�"�γ�Lܳ ""Po�`0@�q���+Ec���d�<��\�r����Y��^9�|�o �������hoZd�g0�fx��NOSO��T��0����"v��Y:�CSL]�Y({��ɘ3��W�V��^�����AX>�@Ap6:û ˚/��p4<�@0?Mg�Y��7,����!�A ֟ձb�% Jn�yi�_Q�5��Ѻ*IM���)�C
/O^�8ϋe^�t���s���<z�p��0�H� �l��dX�
V :W�\��y�Q��7��>9Ӱ �=�����>�n��n�t"�ǿ(���{�˃��Ӵ��FH]��Wް����N�a���liݸ��w ��?z~��"���Ǚ7?��$]�K��"�謿��Xvn���I����2׵�Q޹�%s�Hc�ge�J�7���#��h~��� S8��H�s�_�� ܨ������VF�OQ��}n�?��<�L��ݧ��a���X�0�=@������bBi���˃��v}��Hd��W%dV�/F�+�1f9�§�gZ��$�����D�ҹJ�����̝���N�dȡVd!h5bo�%K��H~�������A�o�s"�7���Hi��(���i�<
�( VHLS�m�)��5.�&��K�_��i��,��� ����,������U�d�_���=I&����'��R`u�+�Z�+�s�V�Uh ��m��ל5��b�[����g��j�0ب����8�
��\��F����B��2����a��c��� ��5o��݄�3ʓI'e~�I�9VL���8��E�'\�=v�Q�$mﰁu�u�}����&ֵ��0��J��Y8&`�Y���z���<���NF0 Yg����JՙJ�\��]��B
������ ������s���6q�D���Z�i>�[�D0<���E���럞N�z��H2b-�3�?�J �2s�5?v����z�=/���1�P;��D;�.UE��l��1#����#)�v{�M�ʫi@�yrH��^��d ����`�_ψF��[���R{6�~ɛT)E�J�~��\��(�^u����몔�J��ެ��u=b/|y>����˛h� s�D�Ԩـ���G �Ul��Ni���I��@�Ƨ����~k�&Vx��5���G^��^�"ۃ����y�:a5E�dmF2K��M����Ug���-6�uI�z4K��t�F鉘��Ȗ��dv�^��-��-�>��b.�>��`V� �������jR�r<s���ω�D`V��q�Y�ʨ���*�Ì�N#ov�Y
���tT���i���{�~������ޔ�:x�P�a栭��������i��_IEND�B`�j
"
@�?�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Xͮ�D����$W�@�-j T]!$6���'�Ux ��a�W@����X���R����$�mn�1ߙ��s��6�l�u�����o��ط�7��4���J�x�䛺��ʷ+��h۾���(�+߮|g�m�.z{��m߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g�m�.z{ڷm߶}g��m׾���������q�������F�&Z����>o@���9�n����|����|���������ۏ��8�����׵ ���򗿿�<K����x����P����`����1��݋4�����+��6��G7�E�H���Y2;N�w������OGx�&i��=��f~��ܣ8�:�� ��{!�:Nn��?=� �/� �� ���?>�%�EP���A�q7~v�?�͵�����X��ߣG��R*�*���KB�/��.]r|�x���ePA� ��"_fK�1X,�%�A�[�'y��7��y��]F��Mje"�%@���lET����X��#r���E�<��<���i��sh�<�=|��wG�9��G�b=<�آ��[j[�|*��ّY��^ ��B�"q¡�����q6���ϔ?ܘ�G�����8 �3 �BhORn���1�̕=���P0q���a ��Ģ#����/��Φ'��ω�"�γ�Lܳ ""Po�`0@�q���+Ec���d�<��\�r����Y��^9�|�o �������hoZd�g0�fx��NOSO��T��0����"v��Y:�CSL]�Y({��ɘ3��W�V��^�����AX>�@Ap6:û ˚/��p4<�@0?Mg�Y��7,����!�A ֟ձb�% Jn�yi�_Q�5��Ѻ*IM���)�C
/O^�8ϋe^�t���s���<z�p��0�H� �l��dX�
V :W�\��y�Q��7��>9Ӱ �=�����>�n��n�t"�ǿ(���{�˃��Ӵ��FH]��Wް����N�a���liݸ��w ��?z~��"���Ǚ7?��$]�K��"�謿��Xvn���I����2׵�Q޹�%s�Hc�ge�J�7���#��h~��� S8��H�s�_�� ܨ������VF�OQ��}n�?��<�L��ݧ��a���X�0�=@������bBi���˃��v}��Hd��W%dV�/F�+�1f9�§�gZ��$�����D�ҹJ�����̝���N�dȡVd!h5bo�%K��H~�������A�o�s"�7���Hi��(���i�<
�( VHLS�m�)��5.�&��K�_��i��,��� ����,������U�d�_���=I&����'��R`u�+�Z�+�s�V�Uh ��m��ל5��b�[����g��j�0ب����8�
��\��F����B��2����a��c��� ��5o��݄�3ʓI'e~�I�9VL���8��E�'\�=v�Q�$mﰁu�u�}����&ֵ��0��J��Y8&`�Y���z���<���NF0 Yg����JՙJ�\��]��B
������ ������s���6q�D���Z�i>�[�D0<���E���럞N�z��H2b-�3�?�J �2s�5?v����z�=/���1�P;��D;�.UE��l��1#����#)�v{�M�ʫi@�yrH��^��d ����`�_ψF��[���R{6�~ɛT)E�J�~��\��(�^u����몔�J��ެ��u=b/|y>����˛h� s�D�Ԩـ���G �Ul��Ni���I��@�Ƨ����~k�&Vx��5���G^��^�"ۃ����y�:a5E�dmF2K��M����Ug���-6�uI�z4K��t�F鉘��Ȗ��dv�^��-��-�>��b.�>��`V� �������jR�r<s���ω�D`V��q�Y�ʨ���*�Ì�N#ov�Y
���tT���i���{�~������ޔ�:x�P�a栭��������i��_IEND�B`�j
"
�?�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�n\E�IF��� X�����k�� ;~�=_6Y@/ $�x,#�P�����OUw�nj��-n�"�z�:U�n�뉗_|��b��Gsκ�����v�۝��ڱ��M�۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~�x����__}������◿��k��]^^�-�����]�k!�^��ln�����.�Ƿ��/�G��C�ڷ ��JL߄���z/�c��{8�i]G���/�������Ez�z�K_~���G�=��LMQ����?����O�럯7�l^���Tj-�Q~���o��o���v� m��������ꇫ����Pp�.�S�_r��ψ�����b/4���H�?y��L�2fONO(uq~�*�7R?O�)����$�
�@V���^$'�呍(ȥl���)�gg|?�S'���>�r׊l�l��
`T��[R%�8�t�"�ɐ��6ⶋ�͆��S��҇ H9�� ��tJ&�q�!% i�,9�.����,��D\�Z��P� 76�!&�ώ�m- %bI�^8ڋ1qP&$"�%��l�X@��S�q IF���`I o �_�H�vPsT��$����X�}� S~�9�*Q(a�Gia���*$����)cGM�t5@DZOsXC)m̪Jq4�!<��;v�*�|P[x����Ql�����`�2��MJ�%y1�h��[^,A��ú1%O��ވe*5��;��M:��R�$��-���@����q�Pm���ݼ�9ҧ�H�au"c���� �,H�p��8j�V������_F�fHJ�)I��-ϖ@Jd�`�>i�t�<:6�&��/�\R)*���" ��zsq��{j�}0O� ���0J��b��u��b�I ���u��# �H �dy"oh�yDj�pP�>l 8^�9�l�1��c��D"��T�#`�RT�K� $�Sci�I��.iV� �#T�PX�K#�nߧT)�0{��L�Qo)�F�B@Ks=7�H���3��l��EH�s?/~i�4'����g�m����2͡���1�B��J�dM�{>Ϳ����*�?mL(�-��88!+$B����͢W�t�ʴ�`H�WGT��q�pP� ˈF&)�pok�S>5�C
U@�:P�*y�!�D�����a��)�����C~Vd�#Ym(���2��ӝGv�L��:�r��u��'ݍ^Җ-������y��^����kz�k�u'�z�į##t$�3
�"�f�Ш7uo3����`V���V�oxY�v(���e�n;j�fq��)�<�C���>+�m�;���C�C춋��wT���7��i��A�K 삟y�ӹ�bw:���c9ԕ9��=��&\��(o{;e��͝�}�|��(�۪Ц���3�3��܅�v��m�v��T��!���Q��s�.���� ���7�˲K�ڱ�� �vQ����w{;[�iS�hgSU�YD�"��y��� �IEND�B`�j
"
�?�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�n\E�IF��� X�����k�� ;~�=_6Y@/ $�x,#�P�����OUw�nj��-n�"�z�:U�n�뉗_|��b��Gsκ�����v�۝��ڱ��M�۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~�x����__}������◿��k��]^^�-�����]�k!�^��ln�����.�Ƿ��/�G��C�ڷ ��JL߄���z/�c��{8�i]G���/�������Ez�z�K_~���G�=��LMQ����?����O�럯7�l^���Tj-�Q~���o��o���v� m��������ꇫ����Pp�.�S�_r��ψ�����b/4���H�?y��L�2fONO(uq~�*�7R?O�)����$�
�@V���^$'�呍(ȥl���)�gg|?�S'���>�r׊l�l��
`T��[R%�8�t�"�ɐ��6ⶋ�͆��S��҇ H9�� ��tJ&�q�!% i�,9�.����,��D\�Z��P� 76�!&�ώ�m- %bI�^8ڋ1qP&$"�%��l�X@��S�q IF���`I o �_�H�vPsT��$����X�}� S~�9�*Q(a�Gia���*$����)cGM�t5@DZOsXC)m̪Jq4�!<��;v�*�|P[x����Ql�����`�2��MJ�%y1�h��[^,A��ú1%O��ވe*5��;��M:��R�$��-���@����q�Pm���ݼ�9ҧ�H�au"c���� �,H�p��8j�V������_F�fHJ�)I��-ϖ@Jd�`�>i�t�<:6�&��/�\R)*���" ��zsq��{j�}0O� ���0J��b��u��b�I ���u��# �H �dy"oh�yDj�pP�>l 8^�9�l�1��c��D"��T�#`�RT�K� $�Sci�I��.iV� �#T�PX�K#�nߧT)�0{��L�Qo)�F�B@Ks=7�H���3��l��EH�s?/~i�4'����g�m����2͡���1�B��J�dM�{>Ϳ����*�?mL(�-��88!+$B����͢W�t�ʴ�`H�WGT��q�pP� ˈF&)�pok�S>5�C
U@�:P�*y�!�D�����a��)�����C~Vd�#Ym(���2��ӝGv�L��:�r��u��'ݍ^Җ-������y��^����kz�k�u'�z�į##t$�3
�"�f�Ш7uo3����`V���V�oxY�v(���e�n;j�fq��)�<�C���>+�m�;���C�C춋��wT���7��i��A�K 삟y�ӹ�bw:���c9ԕ9��=��&\��(o{;e��͝�}�|��(�۪Ц���3�3��܅�v��m�v��T��!���Q��s�.���� ���7�˲K�ڱ�� �vQ����w{;[�iS�hgSU�YD�"��y��� �IEND�B`�j
"
�?�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�n\E�IF��� X�����k�� ;~�=_6Y@/ $�x,#�P�����OUw�nj��-n�"�z�:U�n�뉗_|��b��Gsκ�����v�۝��ڱ��M�۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~�x����__}������◿��k��]^^�-�����]�k!�^��ln�����.�Ƿ��/�G��C�ڷ ��JL߄���z/�c��{8�i]G���/�������Ez�z�K_~���G�=��LMQ����?����O�럯7�l^���Tj-�Q~���o��o���v� m��������ꇫ����Pp�.�S�_r��ψ�����b/4���H�?y��L�2fONO(uq~�*�7R?O�)����$�
�@V���^$'�呍(ȥl���)�gg|?�S'���>�r׊l�l��
`T��[R%�8�t�"�ɐ��6ⶋ�͆��S��҇ H9�� ��tJ&�q�!% i�,9�.����,��D\�Z��P� 76�!&�ώ�m- %bI�^8ڋ1qP&$"�%��l�X@��S�q IF���`I o �_�H�vPsT��$����X�}� S~�9�*Q(a�Gia���*$����)cGM�t5@DZOsXC)m̪Jq4�!<��;v�*�|P[x����Ql�����`�2��MJ�%y1�h��[^,A��ú1%O��ވe*5��;��M:��R�$��-���@����q�Pm���ݼ�9ҧ�H�au"c���� �,H�p��8j�V������_F�fHJ�)I��-ϖ@Jd�`�>i�t�<:6�&��/�\R)*���" ��zsq��{j�}0O� ���0J��b��u��b�I ���u��# �H �dy"oh�yDj�pP�>l 8^�9�l�1��c��D"��T�#`�RT�K� $�Sci�I��.iV� �#T�PX�K#�nߧT)�0{��L�Qo)�F�B@Ks=7�H���3��l��EH�s?/~i�4'����g�m����2͡���1�B��J�dM�{>Ϳ����*�?mL(�-��88!+$B����͢W�t�ʴ�`H�WGT��q�pP� ˈF&)�pok�S>5�C
U@�:P�*y�!�D�����a��)�����C~Vd�#Ym(���2��ӝGv�L��:�r��u��'ݍ^Җ-������y��^����kz�k�u'�z�į##t$�3
�"�f�Ш7uo3����`V���V�oxY�v(���e�n;j�fq��)�<�C���>+�m�;���C�C춋��wT���7��i��A�K 삟y�ӹ�bw:���c9ԕ9��=��&\��(o{;e��͝�}�|��(�۪Ц���3�3��܅�v��m�v��T��!���Q��s�.���� ���7�˲K�ڱ�� �vQ����w{;[�iS�hgSU�YD�"��y��� �IEND�B`�j
"
�?�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�n\E�IF��� X�����k�� ;~�=_6Y@/ $�x,#�P�����OUw�nj��-n�"�z�:U�n�뉗_|��b��Gsκ�����v�۝��ڱ��M�۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~�x����__}������◿��k��]^^�-�����]�k!�^��ln�����.�Ƿ��/�G��C�ڷ ��JL߄���z/�c��{8�i]G���/�������Ez�z�K_~���G�=��LMQ����?����O�럯7�l^���Tj-�Q~���o��o���v� m��������ꇫ����Pp�.�S�_r��ψ�����b/4���H�?y��L�2fONO(uq~�*�7R?O�)����$�
�@V���^$'�呍(ȥl���)�gg|?�S'���>�r׊l�l��
`T��[R%�8�t�"�ɐ��6ⶋ�͆��S��҇ H9�� ��tJ&�q�!% i�,9�.����,��D\�Z��P� 76�!&�ώ�m- %bI�^8ڋ1qP&$"�%��l�X@��S�q IF���`I o �_�H�vPsT��$����X�}� S~�9�*Q(a�Gia���*$����)cGM�t5@DZOsXC)m̪Jq4�!<��;v�*�|P[x����Ql�����`�2��MJ�%y1�h��[^,A��ú1%O��ވe*5��;��M:��R�$��-���@����q�Pm���ݼ�9ҧ�H�au"c���� �,H�p��8j�V������_F�fHJ�)I��-ϖ@Jd�`�>i�t�<:6�&��/�\R)*���" ��zsq��{j�}0O� ���0J��b��u��b�I ���u��# �H �dy"oh�yDj�pP�>l 8^�9�l�1��c��D"��T�#`�RT�K� $�Sci�I��.iV� �#T�PX�K#�nߧT)�0{��L�Qo)�F�B@Ks=7�H���3��l��EH�s?/~i�4'����g�m����2͡���1�B��J�dM�{>Ϳ����*�?mL(�-��88!+$B����͢W�t�ʴ�`H�WGT��q�pP� ˈF&)�pok�S>5�C
U@�:P�*y�!�D�����a��)�����C~Vd�#Ym(���2��ӝGv�L��:�r��u��'ݍ^Җ-������y��^����kz�k�u'�z�į##t$�3
�"�f�Ш7uo3����`V���V�oxY�v(���e�n;j�fq��)�<�C���>+�m�;���C�C춋��wT���7��i��A�K 삟y�ӹ�bw:���c9ԕ9��=��&\��(o{;e��͝�}�|��(�۪Ц���3�3��܅�v��m�v��T��!���Q��s�.���� ���7�˲K�ڱ�� �vQ����w{;[�iS�hgSU�YD�"��y��� �IEND�B`�j
"
�?�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�X�n\E�IF��� X�����k�� ;~�=_6Y@/ $�x,#�P�����OUw�nj��-n�"�z�:U�n�뉗_|��b��Gsκ�����v�۝��ڱ��M�۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~�x����__}������◿��k��]^^�-�����]�k!�^��ln�����.�Ƿ��/�G��C�ڷ ��JL߄���z/�c��{8�i]G���/�������Ez�z�K_~���G�=��LMQ����?����O�럯7�l^���Tj-�Q~���o��o���v� m��������ꇫ����Pp�.�S�_r��ψ�����b/4���H�?y��L�2fONO(uq~�*�7R?O�)����$�
�@V���^$'�呍(ȥl���)�gg|?�S'���>�r׊l�l��
`T��[R%�8�t�"�ɐ��6ⶋ�͆��S��҇ H9�� ��tJ&�q�!% i�,9�.����,��D\�Z��P� 76�!&�ώ�m- %bI�^8ڋ1qP&$"�%��l�X@��S�q IF���`I o �_�H�vPsT��$����X�}� S~�9�*Q(a�Gia���*$����)cGM�t5@DZOsXC)m̪Jq4�!<��;v�*�|P[x����Ql�����`�2��MJ�%y1�h��[^,A��ú1%O��ވe*5��;��M:��R�$��-���@����q�Pm���ݼ�9ҧ�H�au"c���� �,H�p��8j�V������_F�fHJ�)I��-ϖ@Jd�`�>i�t�<:6�&��/�\R)*���" ��zsq��{j�}0O� ���0J��b��u��b�I ���u��# �H �dy"oh�yDj�pP�>l 8^�9�l�1��c��D"��T�#`�RT�K� $�Sci�I��.iV� �#T�PX�K#�nߧT)�0{��L�Qo)�F�B@Ks=7�H���3��l��EH�s?/~i�4'����g�m����2͡���1�B��J�dM�{>Ϳ����*�?mL(�-��88!+$B����͢W�t�ʴ�`H�WGT��q�pP� ˈF&)�pok�S>5�C
U@�:P�*y�!�D�����a��)�����C~Vd�#Ym(���2��ӝGv�L��:�r��u��'ݍ^Җ-������y��^����kz�k�u'�z�į##t$�3
�"�f�Ш7uo3����`V���V�oxY�v(���e�n;j�fq��)�<�C���>+�m�;���C�C춋��wT���7��i��A�K 삟y�ӹ�bw:���c9ԕ9��=��&\��(o{;e��͝�}�|��(�۪Ц���3�3��܅�v��m�v��T��!���Q��s�.���� ���7�˲K�ڱ�� �vQ����w{;[�iS�hgSU�YD�"��y��� �IEND�B`�j
"
@@�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y1r$7 Ժ�"m��WN�������p��p��Y�8�*�A��93������V��M��pV��/������7{m��=����ϝ�;����~���-ϝ�;����~��>/�y�����
�c��M���<����p�������~�y�w����p��m?��<�;\��������O?���d?��{��.�kV����U�����J=== t���n�Թ��^��;�Q�񼛷������;�Wd�����ߝ�緿���]������D^T����;��^�b�&�� }yyq:���ϟ�m�����1r���* ��"�5c���h�g�%��P!A���S׿)���Q���[�/s��m ���ӧ���U.Hg1�B4i@�e��s�1���n�S�u?Pq�[��z�
�h��8eX$/��t�����$��or���x���Z
Y��e0� ���� �S<#��Mb�������������8(��r���@$z2ul ��}}Խ.([$��I��('/|��l� F�!��3�{�J�>����l

}�M�LU�BK:�R��b�f����59M���?<<ʣn�_�F0?T�b���b���d*P�z�S�o��*�O�*9F�A��G��o���̈�J��Ki43�L�5�(�e��Ҹu�c��\0\�Ǟ����.��h�7�P)ʎ'��:J�0��2���˻���O�K�z�@C�`��5��8��k�#��؇G]:HO�϶����*�)!�z�+�d�r�w��u����n)`�i�Z;(�jv��a���b�Pj��C��=��,��01g��A&s�Lٶ7D.�E[r4&�p��:���j��;�N���}T���g�=\�C,$�@KC��u�s��<�%��ݙg0�nģ7���(ŷ�A���K�"Ȁe)�؇|^�6) �� ��-���Y*�{�/89ϋ��[�=O�m{(����<Y�nu�29���I��$�� R�x���}l�d-��Hb��h���j����Ԑk�:��X�� &ŀ��5`+��;ۓCٍ5�zB��?�TWA$�ů��[L/a>&�<J1s��EQҘ���N�O�t4<=|��`'�T�T1W�H��kr�e`�L���V����U� %/#�vGg8�50�l:�h-�yNr��T�B8����P s� �3v��Z�T�����W�,�s���p�F��o�F ����k�Rv�%�v�1!z^����,��/�b!ʅ��'Qә�bE$z�i��a�F07���;]�2]�Ȩ�<�#>f\�Up�q{p8ZLpX �l���%/��E5/#D���]�cD�"����n�N����pԺ��X %'���򖍅�P�������7vjB��XKW���ܔ�` 5����(�ml18��nG<z������4���1����u)-�,�QS�=ܔj�b7�'�������OW���0�
�0o#8�����1���4? �I�[8�w���cs:!��B���Hf_�*�������<|17�[N��IEND�B`�j
"
@@�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y1r$7 Ժ�"m��WN�������p��p��Y�8�*�A��93������V��M��pV��/������7{m��=����ϝ�;����~���-ϝ�;����~��>/�y�����
�c��M���<����p�������~�y�w����p��m?��<�;\��������O?���d?��{��.�kV����U�����J=== t���n�Թ��^��;�Q�񼛷������;�Wd�����ߝ�緿���]������D^T����;��^�b�&�� }yyq:���ϟ�m�����1r���* ��"�5c���h�g�%��P!A���S׿)���Q���[�/s��m ���ӧ���U.Hg1�B4i@�e��s�1���n�S�u?Pq�[��z�
�h��8eX$/��t�����$��or���x���Z
Y��e0� ���� �S<#��Mb�������������8(��r���@$z2ul ��}}Խ.([$��I��('/|��l� F�!��3�{�J�>����l

}�M�LU�BK:�R��b�f����59M���?<<ʣn�_�F0?T�b���b���d*P�z�S�o��*�O�*9F�A��G��o���̈�J��Ki43�L�5�(�e��Ҹu�c��\0\�Ǟ����.��h�7�P)ʎ'��:J�0��2���˻���O�K�z�@C�`��5��8��k�#��؇G]:HO�϶����*�)!�z�+�d�r�w��u����n)`�i�Z;(�jv��a���b�Pj��C��=��,��01g��A&s�Lٶ7D.�E[r4&�p��:���j��;�N���}T���g�=\�C,$�@KC��u�s��<�%��ݙg0�nģ7���(ŷ�A���K�"Ȁe)�؇|^�6) �� ��-���Y*�{�/89ϋ��[�=O�m{(����<Y�nu�29���I��$�� R�x���}l�d-��Hb��h���j����Ԑk�:��X�� &ŀ��5`+��;ۓCٍ5�zB��?�TWA$�ů��[L/a>&�<J1s��EQҘ���N�O�t4<=|��`'�T�T1W�H��kr�e`�L���V����U� %/#�vGg8�50�l:�h-�yNr��T�B8����P s� �3v��Z�T�����W�,�s���p�F��o�F ����k�Rv�%�v�1!z^����,��/�b!ʅ��'Qә�bE$z�i��a�F07���;]�2]�Ȩ�<�#>f\�Up�q{p8ZLpX �l���%/��E5/#D���]�cD�"����n�N����pԺ��X %'���򖍅�P�������7vjB��XKW���ܔ�` 5����(�ml18��nG<z������4���1����u)-�,�QS�=ܔj�b7�'�������OW���0�
�0o#8�����1���4? �I�[8�w���cs:!��B���Hf_�*�������<|17�[N��IEND�B`�j
"
@@�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y1r$7 Ժ�"m��WN�������p��p��Y�8�*�A��93������V��M��pV��/������7{m��=����ϝ�;����~���-ϝ�;����~��>/�y�����
�c��M���<����p�������~�y�w����p��m?��<�;\��������O?���d?��{��.�kV����U�����J=== t���n�Թ��^��;�Q�񼛷������;�Wd�����ߝ�緿���]������D^T����;��^�b�&�� }yyq:���ϟ�m�����1r���* ��"�5c���h�g�%��P!A���S׿)���Q���[�/s��m ���ӧ���U.Hg1�B4i@�e��s�1���n�S�u?Pq�[��z�
�h��8eX$/��t�����$��or���x���Z
Y��e0� ���� �S<#��Mb�������������8(��r���@$z2ul ��}}Խ.([$��I��('/|��l� F�!��3�{�J�>����l

}�M�LU�BK:�R��b�f����59M���?<<ʣn�_�F0?T�b���b���d*P�z�S�o��*�O�*9F�A��G��o���̈�J��Ki43�L�5�(�e��Ҹu�c��\0\�Ǟ����.��h�7�P)ʎ'��:J�0��2���˻���O�K�z�@C�`��5��8��k�#��؇G]:HO�϶����*�)!�z�+�d�r�w��u����n)`�i�Z;(�jv��a���b�Pj��C��=��,��01g��A&s�Lٶ7D.�E[r4&�p��:���j��;�N���}T���g�=\�C,$�@KC��u�s��<�%��ݙg0�nģ7���(ŷ�A���K�"Ȁe)�؇|^�6) �� ��-���Y*�{�/89ϋ��[�=O�m{(����<Y�nu�29���I��$�� R�x���}l�d-��Hb��h���j����Ԑk�:��X�� &ŀ��5`+��;ۓCٍ5�zB��?�TWA$�ů��[L/a>&�<J1s��EQҘ���N�O�t4<=|��`'�T�T1W�H��kr�e`�L���V����U� %/#�vGg8�50�l:�h-�yNr��T�B8����P s� �3v��Z�T�����W�,�s���p�F��o�F ����k�Rv�%�v�1!z^����,��/�b!ʅ��'Qә�bE$z�i��a�F07���;]�2]�Ȩ�<�#>f\�Up�q{p8ZLpX �l���%/��E5/#D���]�cD�"����n�N����pԺ��X %'���򖍅�P�������7vjB��XKW���ܔ�` 5����(�ml18��nG<z������4���1����u)-�,�QS�=ܔj�b7�'�������OW���0�
�0o#8�����1���4? �I�[8�w���cs:!��B���Hf_�*�������<|17�[N��IEND�B`�j
"
@@�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y1r$7 Ժ�"m��WN�������p��p��Y�8�*�A��93������V��M��pV��/������7{m��=����ϝ�;����~���-ϝ�;����~��>/�y�����
�c��M���<����p�������~�y�w����p��m?��<�;\��������O?���d?��{��.�kV����U�����J=== t���n�Թ��^��;�Q�񼛷������;�Wd�����ߝ�緿���]������D^T����;��^�b�&�� }yyq:���ϟ�m�����1r���* ��"�5c���h�g�%��P!A���S׿)���Q���[�/s��m ���ӧ���U.Hg1�B4i@�e��s�1���n�S�u?Pq�[��z�
�h��8eX$/��t�����$��or���x���Z
Y��e0� ���� �S<#��Mb�������������8(��r���@$z2ul ��}}Խ.([$��I��('/|��l� F�!��3�{�J�>����l

}�M�LU�BK:�R��b�f����59M���?<<ʣn�_�F0?T�b���b���d*P�z�S�o��*�O�*9F�A��G��o���̈�J��Ki43�L�5�(�e��Ҹu�c��\0\�Ǟ����.��h�7�P)ʎ'��:J�0��2���˻���O�K�z�@C�`��5��8��k�#��؇G]:HO�϶����*�)!�z�+�d�r�w��u����n)`�i�Z;(�jv��a���b�Pj��C��=��,��01g��A&s�Lٶ7D.�E[r4&�p��:���j��;�N���}T���g�=\�C,$�@KC��u�s��<�%��ݙg0�nģ7���(ŷ�A���K�"Ȁe)�؇|^�6) �� ��-���Y*�{�/89ϋ��[�=O�m{(����<Y�nu�29���I��$�� R�x���}l�d-��Hb��h���j����Ԑk�:��X�� &ŀ��5`+��;ۓCٍ5�zB��?�TWA$�ů��[L/a>&�<J1s��EQҘ���N�O�t4<=|��`'�T�T1W�H��kr�e`�L���V����U� %/#�vGg8�50�l:�h-�yNr��T�B8����P s� �3v��Z�T�����W�,�s���p�F��o�F ����k�Rv�%�v�1!z^����,��/�b!ʅ��'Qә�bE$z�i��a�F07���;]�2]�Ȩ�<�#>f\�Up�q{p8ZLpX �l���%/��E5/#D���]�cD�"����n�N����pԺ��X %'���򖍅�P�������7vjB��XKW���ܔ�` 5����(�ml18��nG<z������4���1����u)-�,�QS�=ܔj�b7�'�������OW���0�
�0o#8�����1���4? �I�[8�w���cs:!��B���Hf_�*�������<|17�[N��IEND�B`�j
"
@@�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y1r$7 Ժ�"m��WN�������p��p��Y�8�*�A��93������V��M��pV��/������7{m��=����ϝ�;����~���-ϝ�;����~��>/�y�����
�c��M���<����p�������~�y�w����p��m?��<�;\��������O?���d?��{��.�kV����U�����J=== t���n�Թ��^��;�Q�񼛷������;�Wd�����ߝ�緿���]������D^T����;��^�b�&�� }yyq:���ϟ�m�����1r���* ��"�5c���h�g�%��P!A���S׿)���Q���[�/s��m ���ӧ���U.Hg1�B4i@�e��s�1���n�S�u?Pq�[��z�
�h��8eX$/��t�����$��or���x���Z
Y��e0� ���� �S<#��Mb�������������8(��r���@$z2ul ��}}Խ.([$��I��('/|��l� F�!��3�{�J�>����l

}�M�LU�BK:�R��b�f����59M���?<<ʣn�_�F0?T�b���b���d*P�z�S�o��*�O�*9F�A��G��o���̈�J��Ki43�L�5�(�e��Ҹu�c��\0\�Ǟ����.��h�7�P)ʎ'��:J�0��2���˻���O�K�z�@C�`��5��8��k�#��؇G]:HO�϶����*�)!�z�+�d�r�w��u����n)`�i�Z;(�jv��a���b�Pj��C��=��,��01g��A&s�Lٶ7D.�E[r4&�p��:���j��;�N���}T���g�=\�C,$�@KC��u�s��<�%��ݙg0�nģ7���(ŷ�A���K�"Ȁe)�؇|^�6) �� ��-���Y*�{�/89ϋ��[�=O�m{(����<Y�nu�29���I��$�� R�x���}l�d-��Hb��h���j����Ԑk�:��X�� &ŀ��5`+��;ۓCٍ5�zB��?�TWA$�ů��[L/a>&�<J1s��EQҘ���N�O�t4<=|��`'�T�T1W�H��kr�e`�L���V����U� %/#�vGg8�50�l:�h-�yNr��T�B8����P s� �3v��Z�T�����W�,�s���p�F��o�F ����k�Rv�%�v�1!z^����,��/�b!ʅ��'Qә�bE$z�i��a�F07���;]�2]�Ȩ�<�#>f\�Up�q{p8ZLpX �l���%/��E5/#D���]�cD�"����n�N����pԺ��X %'���򖍅�P�������7vjB��XKW���ܔ�` 5����(�ml18��nG<z������4���1����u)-�,�QS�=ܔj�b7�'�������OW���0�
�0o#8�����1���4? �I�[8�w���cs:!��B���Hf_�*�������<|17�[N��IEND�B`�j
"
�?�?@�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y��U݁�T��d���GP�|!99F������]�
�}������i%���*�t�9}�1wF��}�ÏWs�|0��y�6�\�~�����p���𦇑۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~n��|����M�˯���<'�fH�=l�H��c��D->}YO�V}J�����ooo�`�c�y����LN�>���3�����ɯ������~�ݫ�����ۿ��i]� �>���۩>���w��7������食-tB���~��j�v�~�:ᨉ�����~u�Z�Y����I��r��T��D�^�xA�������hwu��ޖ�^ݼ߾�#_l?����lr*1�G�r����v��ۋc*Z(�!�ai�U'��܀.>^����eL.)&^��L��������
1o1F��b�2wd �]��Mn m6$��<����&䣫$נ�ł�ZJ�z5�<�5O�emp��,���NEPBe��$� �P�eh�/��.�����
�PM����t�� �t i�)��`$�#E�n�6�e
��� ��
}��� ��������r�F���!@Ƞ���ѱ���S���lJ�����0H���E�<�!�Q�rԋ(�� �A��ßժJ�4��Gjfi�� e�iPYM>4<�b񊡒.��!<|��d�:��"�a�,���F@e��`#�cO����r,JH���z�ɰ����X�ҋ��9��~��L��*����:���Ԛj��1p�%|�^<(�Fj7��kđ:5�¡uz2�"���"��,L�0`h�R�9�;��z��)&��D�ǖ�gK(9s�v�cg@wB?�6���gq\����Dg��%���hG?�T2��� �`n0�J{1t��Dd��F ����� ��J���IR -h���H�� u�r��s�I�S\�G��@I�U�G
����8.Q4&�n�_x��� �\� L{u��q��d��R�~�כ�҆8cfL��S���nT
`)��k�`fᙇ����^V#��R?M~ )� X�� j��3��#O^���2�1 ��1�l<Ǖ,��:��|�m�Y�w�P�[�98��|�}����t�r�GФ��+3
E���T8���e�G:���`k\Sh>KCJU��:��,y�!�B��db%BX�,T
!LAZ�!?I2��$7$_x�Ą��I��-;8�`��% ~���ƒ��Fw���yɱu8�$�3�|^�" _9���_��k8J��y#~��I�1#�������m���0}0�Nwa3�/�$O+����<u��m�t
��8z_)i�<���U#݌�#�O�k�1�c췋��wR�4~�ƙR6)�d�삟yǟRy8c9���c���1��5%��D�/�mo���s}Gxh#V���lx��d�X�|F��t#�]�l��0�EKghlJ�?���4iNۅm�^x�����&U�� Ixn:������ږ�Is�e��?��σ���IEND�B`�j
"
�?�?@�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y��U݁�T��d���GP�|!99F������]�
�}������i%���*�t�9}�1wF��}�ÏWs�|0��y�6�\�~�����p���𦇑۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~n��|����M�˯���<'�fH�=l�H��c��D->}YO�V}J�����ooo�`�c�y����LN�>���3�����ɯ������~�ݫ�����ۿ��i]� �>���۩>���w��7������食-tB���~��j�v�~�:ᨉ�����~u�Z�Y����I��r��T��D�^�xA�������hwu��ޖ�^ݼ߾�#_l?����lr*1�G�r����v��ۋc*Z(�!�ai�U'��܀.>^����eL.)&^��L��������
1o1F��b�2wd �]��Mn m6$��<����&䣫$נ�ł�ZJ�z5�<�5O�emp��,���NEPBe��$� �P�eh�/��.�����
�PM����t�� �t i�)��`$�#E�n�6�e
��� ��
}��� ��������r�F���!@Ƞ���ѱ���S���lJ�����0H���E�<�!�Q�rԋ(�� �A��ßժJ�4��Gjfi�� e�iPYM>4<�b񊡒.��!<|��d�:��"�a�,���F@e��`#�cO����r,JH���z�ɰ����X�ҋ��9��~��L��*����:���Ԛj��1p�%|�^<(�Fj7��kđ:5�¡uz2�"���"��,L�0`h�R�9�;��z��)&��D�ǖ�gK(9s�v�cg@wB?�6���gq\����Dg��%���hG?�T2��� �`n0�J{1t��Dd��F ����� ��J���IR -h���H�� u�r��s�I�S\�G��@I�U�G
����8.Q4&�n�_x��� �\� L{u��q��d��R�~�כ�҆8cfL��S���nT
`)��k�`fᙇ����^V#��R?M~ )� X�� j��3��#O^���2�1 ��1�l<Ǖ,��:��|�m�Y�w�P�[�98��|�}����t�r�GФ��+3
E���T8���e�G:���`k\Sh>KCJU��:��,y�!�B��db%BX�,T
!LAZ�!?I2��$7$_x�Ą��I��-;8�`��% ~���ƒ��Fw���yɱu8�$�3�|^�" _9���_��k8J��y#~��I�1#�������m���0}0�Nwa3�/�$O+����<u��m�t
��8z_)i�<���U#݌�#�O�k�1�c췋��wR�4~�ƙR6)�d�삟yǟRy8c9���c���1��5%��D�/�mo���s}Gxh#V���lx��d�X�|F��t#�]�l��0�EKghlJ�?���4iNۅm�^x�����&U�� Ixn:������ږ�Is�e��?��σ���IEND�B`�j
"
�?�?@�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y��U݁�T��d���GP�|!99F������]�
�}������i%���*�t�9}�1wF��}�ÏWs�|0��y�6�\�~�����p���𦇑۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~n��|����M�˯���<'�fH�=l�H��c��D->}YO�V}J�����ooo�`�c�y����LN�>���3�����ɯ������~�ݫ�����ۿ��i]� �>���۩>���w��7������食-tB���~��j�v�~�:ᨉ�����~u�Z�Y����I��r��T��D�^�xA�������hwu��ޖ�^ݼ߾�#_l?����lr*1�G�r����v��ۋc*Z(�!�ai�U'��܀.>^����eL.)&^��L��������
1o1F��b�2wd �]��Mn m6$��<����&䣫$נ�ł�ZJ�z5�<�5O�emp��,���NEPBe��$� �P�eh�/��.�����
�PM����t�� �t i�)��`$�#E�n�6�e
��� ��
}��� ��������r�F���!@Ƞ���ѱ���S���lJ�����0H���E�<�!�Q�rԋ(�� �A��ßժJ�4��Gjfi�� e�iPYM>4<�b񊡒.��!<|��d�:��"�a�,���F@e��`#�cO����r,JH���z�ɰ����X�ҋ��9��~��L��*����:���Ԛj��1p�%|�^<(�Fj7��kđ:5�¡uz2�"���"��,L�0`h�R�9�;��z��)&��D�ǖ�gK(9s�v�cg@wB?�6���gq\����Dg��%���hG?�T2��� �`n0�J{1t��Dd��F ����� ��J���IR -h���H�� u�r��s�I�S\�G��@I�U�G
����8.Q4&�n�_x��� �\� L{u��q��d��R�~�כ�҆8cfL��S���nT
`)��k�`fᙇ����^V#��R?M~ )� X�� j��3��#O^���2�1 ��1�l<Ǖ,��:��|�m�Y�w�P�[�98��|�}����t�r�GФ��+3
E���T8���e�G:���`k\Sh>KCJU��:��,y�!�B��db%BX�,T
!LAZ�!?I2��$7$_x�Ą��I��-;8�`��% ~���ƒ��Fw���yɱu8�$�3�|^�" _9���_��k8J��y#~��I�1#�������m���0}0�Nwa3�/�$O+����<u��m�t
��8z_)i�<���U#݌�#�O�k�1�c췋��wR�4~�ƙR6)�d�삟yǟRy8c9���c���1��5%��D�/�mo���s}Gxh#V���lx��d�X�|F��t#�]�l��0�EKghlJ�?���4iNۅm�^x�����&U�� Ixn:������ږ�Is�e��?��σ���IEND�B`�j
"
�?�?@�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y��U݁�T��d���GP�|!99F������]�
�}������i%���*�t�9}�1wF��}�ÏWs�|0��y�6�\�~�����p���𦇑۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~n��|����M�˯���<'�fH�=l�H��c��D->}YO�V}J�����ooo�`�c�y����LN�>���3�����ɯ������~�ݫ�����ۿ��i]� �>���۩>���w��7������食-tB���~��j�v�~�:ᨉ�����~u�Z�Y����I��r��T��D�^�xA�������hwu��ޖ�^ݼ߾�#_l?����lr*1�G�r����v��ۋc*Z(�!�ai�U'��܀.>^����eL.)&^��L��������
1o1F��b�2wd �]��Mn m6$��<����&䣫$נ�ł�ZJ�z5�<�5O�emp��,���NEPBe��$� �P�eh�/��.�����
�PM����t�� �t i�)��`$�#E�n�6�e
��� ��
}��� ��������r�F���!@Ƞ���ѱ���S���lJ�����0H���E�<�!�Q�rԋ(�� �A��ßժJ�4��Gjfi�� e�iPYM>4<�b񊡒.��!<|��d�:��"�a�,���F@e��`#�cO����r,JH���z�ɰ����X�ҋ��9��~��L��*����:���Ԛj��1p�%|�^<(�Fj7��kđ:5�¡uz2�"���"��,L�0`h�R�9�;��z��)&��D�ǖ�gK(9s�v�cg@wB?�6���gq\����Dg��%���hG?�T2��� �`n0�J{1t��Dd��F ����� ��J���IR -h���H�� u�r��s�I�S\�G��@I�U�G
����8.Q4&�n�_x��� �\� L{u��q��d��R�~�כ�҆8cfL��S���nT
`)��k�`fᙇ����^V#��R?M~ )� X�� j��3��#O^���2�1 ��1�l<Ǖ,��:��|�m�Y�w�P�[�98��|�}����t�r�GФ��+3
E���T8���e�G:���`k\Sh>KCJU��:��,y�!�B��db%BX�,T
!LAZ�!?I2��$7$_x�Ą��I��-;8�`��% ~���ƒ��Fw���yɱu8�$�3�|^�" _9���_��k8J��y#~��I�1#�������m���0}0�Nwa3�/�$O+����<u��m�t
��8z_)i�<���U#݌�#�O�k�1�c췋��wR�4~�ƙR6)�d�삟yǟRy8c9���c���1��5%��D�/�mo���s}Gxh#V���lx��d�X�|F��t#�]�l��0�EKghlJ�?���4iNۅm�^x�����&U�� Ixn:������ږ�Is�e��?��σ���IEND�B`�j
"
�?�?@�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�Y��U݁�T��d���GP�|!99F������]�
�}������i%���*�t�9}�1wF��}�ÏWs�|0��y�6�\�~�����p���𦇑۝ow~�h�~�7���ڱo�~�h�~�7���۱o�~�h�~�7���۱o�~�h�~�7���۱o�~n��|����M�˯���<'�fH�=l�H��c��D->}YO�V}J�����ooo�`�c�y����LN�>���3�����ɯ������~�ݫ�����ۿ��i]� �>���۩>���w��7������食-tB���~��j�v�~�:ᨉ�����~u�Z�Y����I��r��T��D�^�xA�������hwu��ޖ�^ݼ߾�#_l?����lr*1�G�r����v��ۋc*Z(�!�ai�U'��܀.>^����eL.)&^��L��������
1o1F��b�2wd �]��Mn m6$��<����&䣫$נ�ł�ZJ�z5�<�5O�emp��,���NEPBe��$� �P�eh�/��.�����
�PM����t�� �t i�)��`$�#E�n�6�e
��� ��
}��� ��������r�F���!@Ƞ���ѱ���S���lJ�����0H���E�<�!�Q�rԋ(�� �A��ßժJ�4��Gjfi�� e�iPYM>4<�b񊡒.��!<|��d�:��"�a�,���F@e��`#�cO����r,JH���z�ɰ����X�ҋ��9��~��L��*����:���Ԛj��1p�%|�^<(�Fj7��kđ:5�¡uz2�"���"��,L�0`h�R�9�;��z��)&��D�ǖ�gK(9s�v�cg@wB?�6���gq\����Dg��%���hG?�T2��� �`n0�J{1t��Dd��F ����� ��J���IR -h���H�� u�r��s�I�S\�G��@I�U�G
����8.Q4&�n�_x��� �\� L{u��q��d��R�~�כ�҆8cfL��S���nT
`)��k�`fᙇ����^V#��R?M~ )� X�� j��3��#O^���2�1 ��1�l<Ǖ,��:��|�m�Y�w�P�[�98��|�}����t�r�GФ��+3
E���T8���e�G:���`k\Sh>KCJU��:��,y�!�B��db%BX�,T
!LAZ�!?I2��$7$_x�Ą��I��-;8�`��% ~���ƒ��Fw���yɱu8�$�3�|^�" _9���_��k8J��y#~��I�1#�������m���0}0�Nwa3�/�$O+����<u��m�t
��8z_)i�<���U#݌�#�O�k�1�c췋��wR�4~�ƙR6)�d�삟yǟRy8c9���c���1��5%��D�/�mo���s}Gxh#V���lx��d�X�|F��t#�]�l��0�EKghlJ�?���4iNۅm�^x�����&U�� Ixn:������ږ�Is�e��?��σ���IEND�B`�j
"
�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�XˎUuG��h2�`�0� �D>!{���� �Ċ�%!�Nƚl� ��v�=Uuo����3�o/��q�T��붓����^-�z�Tᬻ�_��oO�=�n��>�$�=�����v���� ��v���v����۾�v���v����۾�v���v����۾�����ꊴ������ߟi��m�!=��ɦF4�O;�=D�]5�d��P����)��.//��.[wj�����(���Z��Q~j��J��;���/�x��g~��������|%Ms�c��|���_����//������7�n�o~�l~����i�;��#������y������ʩ�q���<� ����B 7�����E��߾�U�=;;�V���R~��?~|N雛W4��م�JK! +Su��P[�>zċ;?g 9�~�pmq����Y; ����Q�ܑ5�v W�rS �L��n��y����z�,&�S}��Z[)X�:��tG� K@��Ũ*Ra㪨�� (�'��hN������O?%�i���h�;G�K(X� FV�paX5����<�\o�M���E��n�$,X��%@�qtcΈ�2A"1�,���v�5T��zԈ�H2z.�� �����M �d=`5G�QO"I)��hV���ǯԲJ� ���Gia��� d��|e6`��xt�� ���J} �9�:G�N1�|R[x����Qld�����&�PjD�Up� �b Jo6�)y�{�,�J����:�I�g��� Pǀ����M z�I�8i�v���@G� Uk:͠N�X䳃��� Rn�00�Z��9�;�B�@�f��/QԱ%@��H����g �NB�G�����A�K
!E��Y$�<[Bonv�cO-ÅyB�\�s�(Uۋ�����-'5yf���\����c�P���G���A ��%������Q����Z��J�$"�;�:�G ����,� H7�^t����+*��NG�N����F��~x,m sp��4�„Ȕq��B@Ks�}h��|�A�'8�W����S�cA�H3���9�A��̣���+t��=�Lsh��s ����R<Y��=���c`��J�M�'��G� �����)6����z��뺕i��>���4)7�J��H��LR0���~NE`��,����u�U��C@���2f�S
Aq9��Pd�#�6V_x�b��DŽ�t��]:H��[��R~��� LǓ�F/i˖���Cyg>�e/��bš>��x����-����$�gFAU�,�hԛ���vwb�`�����^���}2�,����Na؈���¨�cO?rB��m_�(8j`�;T:ļ]�ϼ�:D���3�mh�b�س ��;�����鸧;��PW�(R���p-R����픥>7wN�m�� .�o�R�"���Ψ�72� [����l� �S����g�'i"�i���� /�66:�d�(˜d�ce#�vQ���{�U7�����Y��"��]�>���IEND�B`�j
"
�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�XˎUuG��h2�`�0� �D>!{���� �Ċ�%!�Nƚl� ��v�=Uuo����3�o/��q�T��붓����^-�z�Tᬻ�_��oO�=�n��>�$�=�����v���� ��v���v����۾�v���v����۾�v���v����۾�����ꊴ������ߟi��m�!=��ɦF4�O;�=D�]5�d��P����)��.//��.[wj�����(���Z��Q~j��J��;���/�x��g~��������|%Ms�c��|���_����//������7�n�o~�l~����i�;��#������y������ʩ�q���<� ����B 7�����E��߾�U�=;;�V���R~��?~|N雛W4��م�JK! +Su��P[�>zċ;?g 9�~�pmq����Y; ����Q�ܑ5�v W�rS �L��n��y����z�,&�S}��Z[)X�:��tG� K@��Ũ*Ra㪨�� (�'��hN������O?%�i���h�;G�K(X� FV�paX5����<�\o�M���E��n�$,X��%@�qtcΈ�2A"1�,���v�5T��zԈ�H2z.�� �����M �d=`5G�QO"I)��hV���ǯԲJ� ���Gia��� d��|e6`��xt�� ���J} �9�:G�N1�|R[x����Qld�����&�PjD�Up� �b Jo6�)y�{�,�J����:�I�g��� Pǀ����M z�I�8i�v���@G� Uk:͠N�X䳃��� Rn�00�Z��9�;�B�@�f��/QԱ%@��H����g �NB�G�����A�K
!E��Y$�<[Bonv�cO-ÅyB�\�s�(Uۋ�����-'5yf���\����c�P���G���A ��%������Q����Z��J�$"�;�:�G ����,� H7�^t����+*��NG�N����F��~x,m sp��4�„Ȕq��B@Ks�}h��|�A�'8�W����S�cA�H3���9�A��̣���+t��=�Lsh��s ����R<Y��=���c`��J�M�'��G� �����)6����z��뺕i��>���4)7�J��H��LR0���~NE`��,����u�U��C@���2f�S
Aq9��Pd�#�6V_x�b��DŽ�t��]:H��[��R~��� LǓ�F/i˖���Cyg>�e/��bš>��x����-����$�gFAU�,�hԛ���vwb�`�����^���}2�,����Na؈���¨�cO?rB��m_�(8j`�;T:ļ]�ϼ�:D���3�mh�b�س ��;�����鸧;��PW�(R���p-R����픥>7wN�m�� .�o�R�"���Ψ�72� [����l� �S����g�'i"�i���� /�66:�d�(˜d�ce#�vQ���{�U7�����Y��"��]�>���IEND�B`�j
"
�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�XˎUuG��h2�`�0� �D>!{���� �Ċ�%!�Nƚl� ��v�=Uuo����3�o/��q�T��붓����^-�z�Tᬻ�_��oO�=�n��>�$�=�����v���� ��v���v����۾�v���v����۾�v���v����۾�����ꊴ������ߟi��m�!=��ɦF4�O;�=D�]5�d��P����)��.//��.[wj�����(���Z��Q~j��J��;���/�x��g~��������|%Ms�c��|���_����//������7�n�o~�l~����i�;��#������y������ʩ�q���<� ����B 7�����E��߾�U�=;;�V���R~��?~|N雛W4��م�JK! +Su��P[�>zċ;?g 9�~�pmq����Y; ����Q�ܑ5�v W�rS �L��n��y����z�,&�S}��Z[)X�:��tG� K@��Ũ*Ra㪨�� (�'��hN������O?%�i���h�;G�K(X� FV�paX5����<�\o�M���E��n�$,X��%@�qtcΈ�2A"1�,���v�5T��zԈ�H2z.�� �����M �d=`5G�QO"I)��hV���ǯԲJ� ���Gia��� d��|e6`��xt�� ���J} �9�:G�N1�|R[x����Qld�����&�PjD�Up� �b Jo6�)y�{�,�J����:�I�g��� Pǀ����M z�I�8i�v���@G� Uk:͠N�X䳃��� Rn�00�Z��9�;�B�@�f��/QԱ%@��H����g �NB�G�����A�K
!E��Y$�<[Bonv�cO-ÅyB�\�s�(Uۋ�����-'5yf���\����c�P���G���A ��%������Q����Z��J�$"�;�:�G ����,� H7�^t����+*��NG�N����F��~x,m sp��4�„Ȕq��B@Ks�}h��|�A�'8�W����S�cA�H3���9�A��̣���+t��=�Lsh��s ����R<Y��=���c`��J�M�'��G� �����)6����z��뺕i��>���4)7�J��H��LR0���~NE`��,����u�U��C@���2f�S
Aq9��Pd�#�6V_x�b��DŽ�t��]:H��[��R~��� LǓ�F/i˖���Cyg>�e/��bš>��x����-����$�gFAU�,�hԛ���vwb�`�����^���}2�,����Na؈���¨�cO?rB��m_�(8j`�;T:ļ]�ϼ�:D���3�mh�b�س ��;�����鸧;��PW�(R���p-R����픥>7wN�m�� .�o�R�"���Ψ�72� [����l� �S����g�'i"�i���� /�66:�d�(˜d�ce#�vQ���{�U7�����Y��"��]�>���IEND�B`�j
"
�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�XˎUuG��h2�`�0� �D>!{���� �Ċ�%!�Nƚl� ��v�=Uuo����3�o/��q�T��붓����^-�z�Tᬻ�_��oO�=�n��>�$�=�����v���� ��v���v����۾�v���v����۾�v���v����۾�����ꊴ������ߟi��m�!=��ɦF4�O;�=D�]5�d��P����)��.//��.[wj�����(���Z��Q~j��J��;���/�x��g~��������|%Ms�c��|���_����//������7�n�o~�l~����i�;��#������y������ʩ�q���<� ����B 7�����E��߾�U�=;;�V���R~��?~|N雛W4��م�JK! +Su��P[�>zċ;?g 9�~�pmq����Y; ����Q�ܑ5�v W�rS �L��n��y����z�,&�S}��Z[)X�:��tG� K@��Ũ*Ra㪨�� (�'��hN������O?%�i���h�;G�K(X� FV�paX5����<�\o�M���E��n�$,X��%@�qtcΈ�2A"1�,���v�5T��zԈ�H2z.�� �����M �d=`5G�QO"I)��hV���ǯԲJ� ���Gia��� d��|e6`��xt�� ���J} �9�:G�N1�|R[x����Qld�����&�PjD�Up� �b Jo6�)y�{�,�J����:�I�g��� Pǀ����M z�I�8i�v���@G� Uk:͠N�X䳃��� Rn�00�Z��9�;�B�@�f��/QԱ%@��H����g �NB�G�����A�K
!E��Y$�<[Bonv�cO-ÅyB�\�s�(Uۋ�����-'5yf���\����c�P���G���A ��%������Q����Z��J�$"�;�:�G ����,� H7�^t����+*��NG�N����F��~x,m sp��4�„Ȕq��B@Ks�}h��|�A�'8�W����S�cA�H3���9�A��̣���+t��=�Lsh��s ����R<Y��=���c`��J�M�'��G� �����)6����z��뺕i��>���4)7�J��H��LR0���~NE`��,����u�U��C@���2f�S
Aq9��Pd�#�6V_x�b��DŽ�t��]:H��[��R~��� LǓ�F/i˖���Cyg>�e/��bš>��x����-����$�gFAU�,�hԛ���vwb�`�����^���}2�,����Na؈���¨�cO?rB��m_�(8j`�;T:ļ]�ϼ�:D���3�mh�b�س ��;�����鸧;��PW�(R���p-R����픥>7wN�m�� .�o�R�"���Ψ�72� [����l� �S����g�'i"�i���� /�66:�d�(˜d�ce#�vQ���{�U7�����Y��"��]�>���IEND�B`�j
"
�?�?�?�
� P‡�������j�
TT� �PNG

IHDRTT� ���IDATx�XˎUuG��h2�`�0� �D>!{���� �Ċ�%!�Nƚl� ��v�=Uuo����3�o/��q�T��붓����^-�z�Tᬻ�_��oO�=�n��>�$�=�����v���� ��v���v����۾�v���v����۾�v���v����۾�����ꊴ������ߟi��m�!=��ɦF4�O;�=D�]5�d��P����)��.//��.[wj�����(���Z��Q~j��J��;���/�x��g~��������|%Ms�c��|���_����//������7�n�o~�l~����i�;��#������y������ʩ�q���<� ����B 7�����E��߾�U�=;;�V���R~��?~|N雛W4��م�JK! +Su��P[�>zċ;?g 9�~�pmq����Y; ����Q�ܑ5�v W�rS �L��n��y����z�,&�S}��Z[)X�:��tG� K@��Ũ*Ra㪨�� (�'��hN������O?%�i���h�;G�K(X� FV�paX5����<�\o�M���E��n�$,X��%@�qtcΈ�2A"1�,���v�5T��zԈ�H2z.�� �����M �d=`5G�QO"I)��hV���ǯԲJ� ���Gia��� d��|e6`��xt�� ���J} �9�:G�N1�|R[x����Qld�����&�PjD�Up� �b Jo6�)y�{�,�J����:�I�g��� Pǀ����M z�I�8i�v���@G� Uk:͠N�X䳃��� Rn�00�Z��9�;�B�@�f��/QԱ%@��H����g �NB�G�����A�K
!E��Y$�<[Bonv�cO-ÅyB�\�s�(Uۋ�����-'5yf���\����c�P���G���A ��%������Q����Z��J�$"�;�:�G ����,� H7�^t����+*��NG�N����F��~x,m sp��4�„Ȕq��B@Ks�}h��|�A�'8�W����S�cA�H3���9�A��̣���+t��=�Lsh��s ����R<Y��=���c`��J�M�'��G� �����)6����z��뺕i��>���4)7�J��H��LR0���~NE`��,����u�U��C@���2f�S
Aq9��Pd�#�6V_x�b��DŽ�t��]:H��[��R~��� LǓ�F/i˖���Cyg>�e/��bš>��x����-����$�gFAU�,�hԛ���vwb�`�����^���}2�,����Na؈���¨�cO?rB��m_�(8j`�;T:ļ]�ϼ�:D���3�mh�b�س ��;�����鸧;��PW�(R���p-R����픥>7wN�m�� .�o�R�"���Ψ�72� [����l� �S����g�'i"�i���� /�66:�d�(˜d�ce#�vQ���{�U7�����Y��"��]�>���IEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Z�nE�I��bG�+BA)�&���y�#$x�9'`��H�8ή����P�S�U��3;��ǑfF#w�|����q���Ά�\j�X�X�Pw��q�؁q�����q�ǝ`Ʊএ�8������p���~�q�؁q����m?��8���8������~�c�?�'��I\��������* қ����M�Z$k�TPI�2Z�kQ����^���#�_�n�^q��F?�}�������u6W�����/fEq
�l�f�w�WU�{ˬ���'��~�+Z]�ݟ�}����W�z.����p���b1_,J~���>r������_7��Ԛk���̳��/���y{*�W���LJ�/�O�z��"+��r^��eyo��9���̿����/_L������ӽϋ��`� ����� .<����ѝ;�c�s�`?�r�������G�TQΦg���WR�9V�.�e���������8J���3��g2��
�k6�Nn���Z���6�Fq�j{{ �;�v�������
�{���'����Ln>ŝ����}��C QQ�9�-1���̀�G�� �R�����g�I��^�wߜ����jQ-���m���,��/N��www�ъ�����Z�
8������<���-,�G��������P[��!ѐC��m�(�s�l�A�Z�ٙ'����s^;�lD{rt
�l>�y<���+���~�n��h����ZPB ���h���X��i'�ݸ�^}�a��7n�_�Ώ�7�a��ڽFP�hd$�HLX1!������f�@сs����ӗ7�߄�O�9�x�ލ=���f��Rl���PFFL-E@���X���|2X'(=+��ك��W=�hi����(F�4K�ӈ�����)�L�*#�����l�%~��*~إՈ//E�����Q|Д$LwR�������M���:4!�i)ђ�W��XF��,E@�E(^,��Ym�}]1��%���*��N6��(��o��b�ġR����w&���d�j�35X�Q[���W`B҂
C�v����~$^�Y��J��)�;�(x�#[`k{]��C���i V�4�:���+�[�H!z��[��KaLI>,�U�La��ڄQp_n�����|�p�$��2qBeW���0��M_Ď(Lu�� \/K��A�KYe,9��ҁ$Rk��2ӵ#1�b���M[�����C��UX�� J-5HG�Q�w��l=��C�Kv�E:$�["�b��v 2q�Ł�j�tFJ�c�|h�xb�X�:j���j?�b���#����*��Z�(PT������������I+F�ߴ���� °ځ�tB9����y����&Z��I�ʅK#A��Sa�V:�(U�`(�H ^�Z�����ʀ�v��Xх'v԰v6�wS��HN$U�/��{$�u�� /p�!�uBZ�c�F�j�����z� O<��t�z�6���a`��3�I�gL���^�_��w��Ɖ�v���3"W���r�D:���8�)��a/���� �8B�vƦ�It��) :b�u� Uu��}��}`�Z�o�n�.U�j�ϼqu��/�@��6#�F��x� �K�v���aL3�qR]�I� �„p�w���Y�������#�_c#�ⵕ  |���daG{��M#\�Z���eUZ�e���j��z�S� ϱZ֚\g`��U�Ɉ��8ӋR����:���V]!d�(N#��?��˙$�%JIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Z�nE�I��bG�+BA)�&���y�#$x�9'`��H�8ή����P�S�U��3;��ǑfF#w�|����q���Ά�\j�X�X�Pw��q�؁q�����q�ǝ`Ʊএ�8������p���~�q�؁q����m?��8���8������~�c�?�'��I\��������* қ����M�Z$k�TPI�2Z�kQ����^���#�_�n�^q��F?�}�������u6W�����/fEq
�l�f�w�WU�{ˬ���'��~�+Z]�ݟ�}����W�z.����p���b1_,J~���>r������_7��Ԛk���̳��/���y{*�W���LJ�/�O�z��"+��r^��eyo��9���̿����/_L������ӽϋ��`� ����� .<����ѝ;�c�s�`?�r�������G�TQΦg���WR�9V�.�e���������8J���3��g2��
�k6�Nn���Z���6�Fq�j{{ �;�v�������
�{���'����Ln>ŝ����}��C QQ�9�-1���̀�G�� �R�����g�I��^�wߜ����jQ-���m���,��/N��www�ъ�����Z�
8������<���-,�G��������P[��!ѐC��m�(�s�l�A�Z�ٙ'����s^;�lD{rt
�l>�y<���+���~�n��h����ZPB ���h���X��i'�ݸ�^}�a��7n�_�Ώ�7�a��ڽFP�hd$�HLX1!������f�@сs����ӗ7�߄�O�9�x�ލ=���f��Rl���PFFL-E@���X���|2X'(=+��ك��W=�hi����(F�4K�ӈ�����)�L�*#�����l�%~��*~إՈ//E�����Q|Д$LwR�������M���:4!�i)ђ�W��XF��,E@�E(^,��Ym�}]1��%���*��N6��(��o��b�ġR����w&���d�j�35X�Q[���W`B҂
C�v����~$^�Y��J��)�;�(x�#[`k{]��C���i V�4�:���+�[�H!z��[��KaLI>,�U�La��ڄQp_n�����|�p�$��2qBeW���0��M_Ď(Lu�� \/K��A�KYe,9��ҁ$Rk��2ӵ#1�b���M[�����C��UX�� J-5HG�Q�w��l=��C�Kv�E:$�["�b��v 2q�Ł�j�tFJ�c�|h�xb�X�:j���j?�b���#����*��Z�(PT������������I+F�ߴ���� °ځ�tB9����y����&Z��I�ʅK#A��Sa�V:�(U�`(�H ^�Z�����ʀ�v��Xх'v԰v6�wS��HN$U�/��{$�u�� /p�!�uBZ�c�F�j�����z� O<��t�z�6���a`��3�I�gL���^�_��w��Ɖ�v���3"W���r�D:���8�)��a/���� �8B�vƦ�It��) :b�u� Uu��}��}`�Z�o�n�.U�j�ϼqu��/�@��6#�F��x� �K�v���aL3�qR]�I� �„p�w���Y�������#�_c#�ⵕ  |���daG{��M#\�Z���eUZ�e���j��z�S� ϱZ֚\g`��U�Ɉ��8ӋR����:���V]!d�(N#��?��˙$�%JIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Z�nE�I��bG�+BA)�&���y�#$x�9'`��H�8ή����P�S�U��3;��ǑfF#w�|����q���Ά�\j�X�X�Pw��q�؁q�����q�ǝ`Ʊএ�8������p���~�q�؁q����m?��8���8������~�c�?�'��I\��������* қ����M�Z$k�TPI�2Z�kQ����^���#�_�n�^q��F?�}�������u6W�����/fEq
�l�f�w�WU�{ˬ���'��~�+Z]�ݟ�}����W�z.����p���b1_,J~���>r������_7��Ԛk���̳��/���y{*�W���LJ�/�O�z��"+��r^��eyo��9���̿����/_L������ӽϋ��`� ����� .<����ѝ;�c�s�`?�r�������G�TQΦg���WR�9V�.�e���������8J���3��g2��
�k6�Nn���Z���6�Fq�j{{ �;�v�������
�{���'����Ln>ŝ����}��C QQ�9�-1���̀�G�� �R�����g�I��^�wߜ����jQ-���m���,��/N��www�ъ�����Z�
8������<���-,�G��������P[��!ѐC��m�(�s�l�A�Z�ٙ'����s^;�lD{rt
�l>�y<���+���~�n��h����ZPB ���h���X��i'�ݸ�^}�a��7n�_�Ώ�7�a��ڽFP�hd$�HLX1!������f�@сs����ӗ7�߄�O�9�x�ލ=���f��Rl���PFFL-E@���X���|2X'(=+��ك��W=�hi����(F�4K�ӈ�����)�L�*#�����l�%~��*~إՈ//E�����Q|Д$LwR�������M���:4!�i)ђ�W��XF��,E@�E(^,��Ym�}]1��%���*��N6��(��o��b�ġR����w&���d�j�35X�Q[���W`B҂
C�v����~$^�Y��J��)�;�(x�#[`k{]��C���i V�4�:���+�[�H!z��[��KaLI>,�U�La��ڄQp_n�����|�p�$��2qBeW���0��M_Ď(Lu�� \/K��A�KYe,9��ҁ$Rk��2ӵ#1�b���M[�����C��UX�� J-5HG�Q�w��l=��C�Kv�E:$�["�b��v 2q�Ł�j�tFJ�c�|h�xb�X�:j���j?�b���#����*��Z�(PT������������I+F�ߴ���� °ځ�tB9����y����&Z��I�ʅK#A��Sa�V:�(U�`(�H ^�Z�����ʀ�v��Xх'v԰v6�wS��HN$U�/��{$�u�� /p�!�uBZ�c�F�j�����z� O<��t�z�6���a`��3�I�gL���^�_��w��Ɖ�v���3"W���r�D:���8�)��a/���� �8B�vƦ�It��) :b�u� Uu��}��}`�Z�o�n�.U�j�ϼqu��/�@��6#�F��x� �K�v���aL3�qR]�I� �„p�w���Y�������#�_c#�ⵕ  |���daG{��M#\�Z���eUZ�e���j��z�S� ϱZ֚\g`��U�Ɉ��8ӋR����:���V]!d�(N#��?��˙$�%JIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Z�nE�I��bG�+BA)�&���y�#$x�9'`��H�8ή����P�S�U��3;��ǑfF#w�|����q���Ά�\j�X�X�Pw��q�؁q�����q�ǝ`Ʊএ�8������p���~�q�؁q����m?��8���8������~�c�?�'��I\��������* қ����M�Z$k�TPI�2Z�kQ����^���#�_�n�^q��F?�}�������u6W�����/fEq
�l�f�w�WU�{ˬ���'��~�+Z]�ݟ�}����W�z.����p���b1_,J~���>r������_7��Ԛk���̳��/���y{*�W���LJ�/�O�z��"+��r^��eyo��9���̿����/_L������ӽϋ��`� ����� .<����ѝ;�c�s�`?�r�������G�TQΦg���WR�9V�.�e���������8J���3��g2��
�k6�Nn���Z���6�Fq�j{{ �;�v�������
�{���'����Ln>ŝ����}��C QQ�9�-1���̀�G�� �R�����g�I��^�wߜ����jQ-���m���,��/N��www�ъ�����Z�
8������<���-,�G��������P[��!ѐC��m�(�s�l�A�Z�ٙ'����s^;�lD{rt
�l>�y<���+���~�n��h����ZPB ���h���X��i'�ݸ�^}�a��7n�_�Ώ�7�a��ڽFP�hd$�HLX1!������f�@сs����ӗ7�߄�O�9�x�ލ=���f��Rl���PFFL-E@���X���|2X'(=+��ك��W=�hi����(F�4K�ӈ�����)�L�*#�����l�%~��*~إՈ//E�����Q|Д$LwR�������M���:4!�i)ђ�W��XF��,E@�E(^,��Ym�}]1��%���*��N6��(��o��b�ġR����w&���d�j�35X�Q[���W`B҂
C�v����~$^�Y��J��)�;�(x�#[`k{]��C���i V�4�:���+�[�H!z��[��KaLI>,�U�La��ڄQp_n�����|�p�$��2qBeW���0��M_Ď(Lu�� \/K��A�KYe,9��ҁ$Rk��2ӵ#1�b���M[�����C��UX�� J-5HG�Q�w��l=��C�Kv�E:$�["�b��v 2q�Ł�j�tFJ�c�|h�xb�X�:j���j?�b���#����*��Z�(PT������������I+F�ߴ���� °ځ�tB9����y����&Z��I�ʅK#A��Sa�V:�(U�`(�H ^�Z�����ʀ�v��Xх'v԰v6�wS��HN$U�/��{$�u�� /p�!�uBZ�c�F�j�����z� O<��t�z�6���a`��3�I�gL���^�_��w��Ɖ�v���3"W���r�D:���8�)��a/���� �8B�vƦ�It��) :b�u� Uu��}��}`�Z�o�n�.U�j�ϼqu��/�@��6#�F��x� �K�v���aL3�qR]�I� �„p�w���Y�������#�_c#�ⵕ  |���daG{��M#\�Z���eUZ�e���j��z�S� ϱZ֚\g`��U�Ɉ��8ӋR����:���V]!d�(N#��?��˙$�%JIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�Z�nE�I��bG�+BA)�&���y�#$x�9'`��H�8ή����P�S�U��3;��ǑfF#w�|����q���Ά�\j�X�X�Pw��q�؁q�����q�ǝ`Ʊএ�8������p���~�q�؁q����m?��8���8������~�c�?�'��I\��������* қ����M�Z$k�TPI�2Z�kQ����^���#�_�n�^q��F?�}�������u6W�����/fEq
�l�f�w�WU�{ˬ���'��~�+Z]�ݟ�}����W�z.����p���b1_,J~���>r������_7��Ԛk���̳��/���y{*�W���LJ�/�O�z��"+��r^��eyo��9���̿����/_L������ӽϋ��`� ����� .<����ѝ;�c�s�`?�r�������G�TQΦg���WR�9V�.�e���������8J���3��g2��
�k6�Nn���Z���6�Fq�j{{ �;�v�������
�{���'����Ln>ŝ����}��C QQ�9�-1���̀�G�� �R�����g�I��^�wߜ����jQ-���m���,��/N��www�ъ�����Z�
8������<���-,�G��������P[��!ѐC��m�(�s�l�A�Z�ٙ'����s^;�lD{rt
�l>�y<���+���~�n��h����ZPB ���h���X��i'�ݸ�^}�a��7n�_�Ώ�7�a��ڽFP�hd$�HLX1!������f�@сs����ӗ7�߄�O�9�x�ލ=���f��Rl���PFFL-E@���X���|2X'(=+��ك��W=�hi����(F�4K�ӈ�����)�L�*#�����l�%~��*~إՈ//E�����Q|Д$LwR�������M���:4!�i)ђ�W��XF��,E@�E(^,��Ym�}]1��%���*��N6��(��o��b�ġR����w&���d�j�35X�Q[���W`B҂
C�v����~$^�Y��J��)�;�(x�#[`k{]��C���i V�4�:���+�[�H!z��[��KaLI>,�U�La��ڄQp_n�����|�p�$��2qBeW���0��M_Ď(Lu�� \/K��A�KYe,9��ҁ$Rk��2ӵ#1�b���M[�����C��UX�� J-5HG�Q�w��l=��C�Kv�E:$�["�b��v 2q�Ł�j�tFJ�c�|h�xb�X�:j���j?�b���#����*��Z�(PT������������I+F�ߴ���� °ځ�tB9����y����&Z��I�ʅK#A��Sa�V:�(U�`(�H ^�Z�����ʀ�v��Xх'v԰v6�wS��HN$U�/��{$�u�� /p�!�uBZ�c�F�j�����z� O<��t�z�6���a`��3�I�gL���^�_��w��Ɖ�v���3"W���r�D:���8�)��a/���� �8B�vƦ�It��) :b�u� Uu��}��}`�Z�o�n�.U�j�ϼqu��/�@��6#�F��x� �K�v���aL3�qR]�I� �„p�w���Y�������#�_c#�ⵕ  |���daG{��M#\�Z���eUZ�e���j��z�S� ϱZ֚\g`��U�Ɉ��8ӋR����:���V]!d�(N#��?��˙$�%JIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�ZK�E�%��ټ �\,ʕK�q��=?q������ʕ�P�8�1+���;�;TuWuU�􌽎c��L��z|_���=�d�ٗ_e]���j��w�|WW�_�~�;8�~�wp�]����+�� �۾���x����}'�o�.z��۾���@��;���i�o�~�wp�������}����3����{<zx���7��$|�ɭ&���-͜����_?�c���eY?���-7�T����v ��m��/�8yx2=<�}���y�߻}�ȊrR��tyZU���j2�\�.o��p�?�e?�}����lA�b�,������<-�C��>}u�Q�T�f�yV��:���r>E?�f�,����&��t���W>}t����QSA�*ۍo�~����q {������q����?<��Gw���tAYg�}&A(g���Y�Aͬ�W͎<�91�3:�S^36j6&�
�ް�a��ɬ�Y<*�m8���<F�R�lG͇
�}�0?��������ɿ�E�\�~;ߵ]�GEYu���_�ze<�T�"��h�H��w�8����X���ȥ�ޗ�_�x|�/]���O���s\�G|��- ���7o�Mz��E�`��'h�����ܺ�:��Ǹ���E�������� \y<ܡ����4�C�(ժ��B�+���Λv9.����%;���s��Ň�G�^/����09ע(*p�޻)3a����L���� \��"��+�� ��۲|��z����m� 1�CY��ɵ��� B%�xBG�&QH��u������G�]�O�����i�V����=�n��l��q��M
��N�����`�I�����$_p������ݶctqxk� �ntT�&� +��7��b�D �
���� �q�x�%4���M�,4���P�D@{##�
#7��P `�h������=`�E7��8���f �B�I\?�3�D�8� ��A�P]
а�CZ��ڦ�
vI�M�h�n Bpa�D@�GCI��D$6��82��m��%��}!�rq�О��0��J<I�⛅5c�qO{��Z"���yE ]�C�L�CB�`�Dx{#߬�4t�P��w{S�j��4��j`Ń�,�Q��(x&$�С`h ����=~���.��G�A��)�
%
%�ԑu��>��^U�ʋ�R=��F�z꩔/ԏ�E%�n�"\��Ho ��Uy���-[��Q�RV�a}�1pQŞF�����(,��D� ��7zG�W���+8�O)�΍}qB���'� �Jȓ�������򫎣ի_âȡ�)�jnm2PV�=C�@�r �����o����uO�����]��� �[ �M�Q֠(w��ds���!:�I�A�3ρB��Bu&h:��2�!��qHwA��y�#�(t�Ӟc�j��٪PK#��<���@�� sT�EVohm� �'��{Ȥ�KW�5�Qo�·C�ȇ74�n�x<��3��/�F G�H]x�(U>� ���s��L�`i}�}\N����U~Ðz�Y)iD"I}"y �3������0�j�)���!x6V�d����&�:@ӂ��pnc���t��0!YBX���i����L���p����1~�`��<�y�,|���؇qW����W�N�g1��?=��܏q�I"o?e������}&�m�a"�I��b���H覚��l������f����3$s ��7p&H����j�P��ImWCw��"��$��Y��Y�D��'TԔ�60E>��{p��-�ݒ��K� � o^���.O�h���8;ou�<��}��Ρj0~��o�7`+N��5G_�ƴ�����A6:��Ԧ�V�Aj�.����m���N+��V�袩7�8��5[�
qF�% t^i(Dn�e'�N��՚���f�w!��^/#��wEj��k^{-v��,=�J�]nq�,�s; ��hp�9U���� L��6r�IB�l4X���Y4 �i��������
�)�/��ED$(DLIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�ZK�E�%��ټ �\,ʕK�q��=?q������ʕ�P�8�1+���;�;TuWuU�􌽎c��L��z|_���=�d�ٗ_e]���j��w�|WW�_�~�;8�~�wp�]����+�� �۾���x����}'�o�.z��۾���@��;���i�o�~�wp�������}����3����{<zx���7��$|�ɭ&���-͜����_?�c���eY?���-7�T����v ��m��/�8yx2=<�}���y�߻}�ȊrR��tyZU���j2�\�.o��p�?�e?�}����lA�b�,������<-�C��>}u�Q�T�f�yV��:���r>E?�f�,����&��t���W>}t����QSA�*ۍo�~����q {������q����?<��Gw���tAYg�}&A(g���Y�Aͬ�W͎<�91�3:�S^36j6&�
�ް�a��ɬ�Y<*�m8���<F�R�lG͇
�}�0?��������ɿ�E�\�~;ߵ]�GEYu���_�ze<�T�"��h�H��w�8����X���ȥ�ޗ�_�x|�/]���O���s\�G|��- ���7o�Mz��E�`��'h�����ܺ�:��Ǹ���E�������� \y<ܡ����4�C�(ժ��B�+���Λv9.����%;���s��Ň�G�^/����09ע(*p�޻)3a����L���� \��"��+�� ��۲|��z����m� 1�CY��ɵ��� B%�xBG�&QH��u������G�]�O�����i�V����=�n��l��q��M
��N�����`�I�����$_p������ݶctqxk� �ntT�&� +��7��b�D �
���� �q�x�%4���M�,4���P�D@{##�
#7��P `�h������=`�E7��8���f �B�I\?�3�D�8� ��A�P]
а�CZ��ڦ�
vI�M�h�n Bpa�D@�GCI��D$6��82��m��%��}!�rq�О��0��J<I�⛅5c�qO{��Z"���yE ]�C�L�CB�`�Dx{#߬�4t�P��w{S�j��4��j`Ń�,�Q��(x&$�С`h ����=~���.��G�A��)�
%
%�ԑu��>��^U�ʋ�R=��F�z꩔/ԏ�E%�n�"\��Ho ��Uy���-[��Q�RV�a}�1pQŞF�����(,��D� ��7zG�W���+8�O)�΍}qB���'� �Jȓ�������򫎣ի_âȡ�)�jnm2PV�=C�@�r �����o����uO�����]��� �[ �M�Q֠(w��ds���!:�I�A�3ρB��Bu&h:��2�!��qHwA��y�#�(t�Ӟc�j��٪PK#��<���@�� sT�EVohm� �'��{Ȥ�KW�5�Qo�·C�ȇ74�n�x<��3��/�F G�H]x�(U>� ���s��L�`i}�}\N����U~Ðz�Y)iD"I}"y �3������0�j�)���!x6V�d����&�:@ӂ��pnc���t��0!YBX���i����L���p����1~�`��<�y�,|���؇qW����W�N�g1��?=��܏q�I"o?e������}&�m�a"�I��b���H覚��l������f����3$s ��7p&H����j�P��ImWCw��"��$��Y��Y�D��'TԔ�60E>��{p��-�ݒ��K� � o^���.O�h���8;ou�<��}��Ρj0~��o�7`+N��5G_�ƴ�����A6:��Ԧ�V�Aj�.����m���N+��V�袩7�8��5[�
qF�% t^i(Dn�e'�N��՚���f�w!��^/#��wEj��k^{-v��,=�J�]nq�,�s; ��hp�9U���� L��6r�IB�l4X���Y4 �i��������
�)�/��ED$(DLIEND�B`�j
"
@�
�P‡�������j�
TT��PNG

IHDRTT� ���IDATx�ZK�E�%��ټ �\,ʕK�q��=?q������ʕ�P�8�1+���;�;TuWuU�􌽎c��L��z|_���=�d�ٗ_e]���j��w�|WW�_�~�;8�~�wp�]����+�� �۾���x����}'�o�.z��۾���@��;���i�o�~�wp�������}����3����{<zx���7��$|�ɭ&���-͜����_?�c���eY?���-7�T����v ��m��/�8yx2=<�}���y�߻}�ȊrR��tyZU���j2�\�.o��p�?�e?�}����lA�b�,������<-�C��>}u�Q�T�f�yV��:���r>E?�f�,����&��t���W>}t����QSA�*ۍo�~����q {������q����?<��Gw���tAYg�}&A(g���Y�Aͬ�W͎<�91�3:�S^36j6&�
�ް�a��ɬ�Y<*�m8���<F�R�lG͇
�}�0?��������ɿ�E�\�~;ߵ]�GEYu���_�ze<�T�"��h�H��w�8����X���ȥ�ޗ�_�x|�/]���O���s\�G|��- ���7o�Mz��E�`��'h�����ܺ�:��Ǹ���E�������� \y<ܡ����4�C�(ժ��B�+���Λv9.����%;���s��Ň�G�^/����09ע(*p�޻)3a����L���� \��"��+�� ��۲|��z����m� 1�CY��ɵ��� B%�xBG�&QH��u������G�]�O�����i�V����=�n��l��q��M
��N�����`�I�����$_p������ݶctqxk� �ntT�&� +��7��b�D �
���� �q�x�%4���M�,4���P�D@{##�
#7��P `�h������=`�E7��8���f �B�I\?�3�D�8� ��A�P]
а�CZ��ڦ�
vI�M�h�n Bpa�D@�GCI��D$6��82��m��%��}!�rq�О��0��J<I�⛅5c�qO{��Z"���yE ]�C�L�CB�`�Dx{#߬�4t�P��w{S�j��4��j`Ń�,�Q��(x&$�С`h ����=~���.��G�A��)�
%
%�ԑu��>��^U�ʋ�R=��F�z꩔/ԏ�E%�n�"\��Ho ��Uy���-[��Q�RV�a}�1pQŞF�����(,��D� ��7zG�W���+8�O)�΍}qB���'� �Jȓ�������򫎣ի_âȡ�)�jnm2PV�=C�@�r �����o����uO�����]��� �[ �M�Q֠(w��ds���!:�I�A�3ρB��Bu&h:��2�!��qHwA��y�#�(t�Ӟc�j��٪PK#��<���@�� sT�EVohm� �'��{Ȥ�KW�5�Qo�·C�ȇ74�n�x<��3��/�F G�H]x�(U>� ���s��L�`i}�}\N����U~Ðz�Y)iD"I}"y �3������0�j�)���!x6V�d����&�:@ӂ��pnc���t��0!YBX���i����L���p����1~�`��<�y�,|���؇qW����W�N�g1��?=��܏q�I"o?e������}&�m�a"�I��b���H覚��l������f����3$s ��7p&H����j�P��ImWCw��"��$��Y��Y�D��'TԔ�60E>��{p��-�ݒ��K� � o^���.O�h���8;ou�<��}��Ρj0~��o�7`+N��5G_�ƴ�����A6:��Ԧ�V�Aj�.����m���N+��V�袩7�8��5[�
qF�% t^i(Dn�e'�N��՚���f�w!��^/#��wEj��k^{-v��,=�J�]nq�,�s; ��hp�9U���� L��6r�IB�l4X���Y4 �i��������
�)�/��ED$(DLIEND�B`�j
"

0
ml-agents/mlagents/trainers/torch/components/bc/__init__.py

185
ml-agents/mlagents/trainers/torch/components/bc/module.py


from typing import Dict
import numpy as np
import torch
from mlagents.trainers.policy.torch_policy import TorchPolicy
from mlagents.trainers.demo_loader import demo_to_buffer
from mlagents.trainers.settings import BehavioralCloningSettings, ScheduleType
from mlagents.trainers.torch.utils import ModelUtils
class BCModule:
def __init__(
self,
policy: TorchPolicy,
settings: BehavioralCloningSettings,
policy_learning_rate: float,
default_batch_size: int,
default_num_epoch: int,
):
"""
A BC trainer that can be used inline with RL.
:param policy: The policy of the learning model
:param settings: The settings for BehavioralCloning including LR strength, batch_size,
num_epochs, samples_per_update and LR annealing steps.
:param policy_learning_rate: The initial Learning Rate of the policy. Used to set an appropriate learning rate
for the pretrainer.
"""
self.policy = policy
self._anneal_steps = settings.steps
self.current_lr = policy_learning_rate * settings.strength
learning_rate_schedule: ScheduleType = ScheduleType.LINEAR if self._anneal_steps > 0 else ScheduleType.CONSTANT
self.decay_learning_rate = ModelUtils.DecayedValue(
learning_rate_schedule, self.current_lr, 1e-10, self._anneal_steps
)
params = self.policy.actor_critic.parameters()
self.optimizer = torch.optim.Adam(params, lr=self.current_lr)
_, self.demonstration_buffer = demo_to_buffer(
settings.demo_path, policy.sequence_length, policy.behavior_spec
)
self.batch_size = (
settings.batch_size if settings.batch_size else default_batch_size
)
self.num_epoch = settings.num_epoch if settings.num_epoch else default_num_epoch
self.n_sequences = max(
min(self.batch_size, self.demonstration_buffer.num_experiences)
// policy.sequence_length,
1,
)
self.has_updated = False
self.use_recurrent = self.policy.use_recurrent
self.samples_per_update = settings.samples_per_update
def update(self) -> Dict[str, np.ndarray]:
"""
Updates model using buffer.
:param max_batches: The maximum number of batches to use per update.
:return: The loss of the update.
"""
# Don't continue training if the learning rate has reached 0, to reduce training time.
decay_lr = self.decay_learning_rate.get_value(self.policy.get_current_step())
if self.current_lr <= 0:
return {"Losses/Pretraining Loss": 0}
batch_losses = []
possible_demo_batches = (
self.demonstration_buffer.num_experiences // self.n_sequences
)
possible_batches = possible_demo_batches
max_batches = self.samples_per_update // self.n_sequences
n_epoch = self.num_epoch
for _ in range(n_epoch):
self.demonstration_buffer.shuffle(
sequence_length=self.policy.sequence_length
)
if max_batches == 0:
num_batches = possible_batches
else:
num_batches = min(possible_batches, max_batches)
for i in range(num_batches // self.policy.sequence_length):
demo_update_buffer = self.demonstration_buffer
start = i * self.n_sequences * self.policy.sequence_length
end = (i + 1) * self.n_sequences * self.policy.sequence_length
mini_batch_demo = demo_update_buffer.make_mini_batch(start, end)
run_out = self._update_batch(mini_batch_demo, self.n_sequences)
loss = run_out["loss"]
batch_losses.append(loss)
ModelUtils.update_learning_rate(self.optimizer, decay_lr)
self.current_lr = decay_lr
self.has_updated = True
update_stats = {"Losses/Pretraining Loss": np.mean(batch_losses)}
return update_stats
def _behavioral_cloning_loss(self, selected_actions, log_probs, expert_actions):
if self.policy.use_continuous_act:
bc_loss = torch.nn.functional.mse_loss(selected_actions, expert_actions)
else:
log_prob_branches = ModelUtils.break_into_branches(
log_probs, self.policy.act_size
)
bc_loss = torch.mean(
torch.stack(
[
torch.sum(
-torch.nn.functional.log_softmax(log_prob_branch, dim=1)
* expert_actions_branch,
dim=1,
)
for log_prob_branch, expert_actions_branch in zip(
log_prob_branches, expert_actions
)
]
)
)
return bc_loss
def _update_batch(
self, mini_batch_demo: Dict[str, np.ndarray], n_sequences: int
) -> Dict[str, float]:
"""
Helper function for update_batch.
"""
vec_obs = [ModelUtils.list_to_tensor(mini_batch_demo["vector_obs"])]
act_masks = None
if self.policy.use_continuous_act:
expert_actions = ModelUtils.list_to_tensor(mini_batch_demo["actions"])
else:
raw_expert_actions = ModelUtils.list_to_tensor(
mini_batch_demo["actions"], dtype=torch.long
)
expert_actions = ModelUtils.actions_to_onehot(
raw_expert_actions, self.policy.act_size
)
act_masks = ModelUtils.list_to_tensor(
np.ones(
(
self.n_sequences * self.policy.sequence_length,
sum(self.policy.behavior_spec.discrete_action_branches),
),
dtype=np.float32,
)
)
memories = []
if self.policy.use_recurrent:
memories = torch.zeros(
1, self.n_sequences, self.policy.actor_critic.half_mem_size * 2
)
if self.policy.use_vis_obs:
vis_obs = []
for idx, _ in enumerate(
self.policy.actor_critic.network_body.visual_encoders
):
vis_ob = ModelUtils.list_to_tensor(
mini_batch_demo["visual_obs%d" % idx]
)
vis_obs.append(vis_ob)
else:
vis_obs = []
selected_actions, all_log_probs, _, _, _ = self.policy.sample_actions(
vec_obs,
vis_obs,
masks=act_masks,
memories=memories,
seq_len=self.policy.sequence_length,
all_log_probs=True,
)
bc_loss = self._behavioral_cloning_loss(
selected_actions, all_log_probs, expert_actions
)
self.optimizer.zero_grad()
bc_loss.backward()
self.optimizer.step()
run_out = {"loss": bc_loss.detach().cpu().numpy()}
return run_out
正在加载...
取消
保存