From bd390c2adecb5d606c455c9fd7099b674add3109 Mon Sep 17 00:00:00 2001 From: Peter Zhokhov Date: Fri, 19 Oct 2018 17:50:54 -0700 Subject: [PATCH 01/14] updated docstring for deepq --- baselines/deepq/deepq.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/baselines/deepq/deepq.py b/baselines/deepq/deepq.py index c6004b2..c8de92d 100644 --- a/baselines/deepq/deepq.py +++ b/baselines/deepq/deepq.py @@ -124,16 +124,12 @@ def learn(env, ------- env: gym.Env environment to train on - q_func: (tf.Variable, int, str, bool) -> tf.Variable - the model that takes the following inputs: - observation_in: object - the output of observation placeholder - num_actions: int - number of actions - scope: str - reuse: bool - should be passed to outer variable scope - and returns a tensor of shape (batch_size, num_actions) with values of every action. + network: string or a function + neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models + (mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which + will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that) + seed: int or None + prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used. lr: float learning rate for adam optimizer total_timesteps: int From c0fa11a3a730dcf987040ea131461ccd74e7da7b Mon Sep 17 00:00:00 2001 From: pzhokhov Date: Mon, 22 Oct 2018 09:15:04 -0700 Subject: [PATCH 02/14] minor fixes from internal (#665) * sync internal changes. Make ddpg work with vecenvs * B -> nenvs for consistency with other algos, small cleanups * eval_done[d]==True -> eval_done[d] * flake8 and numpy.random.random_integers deprecation warning * Merge branch 'master' of github.com:openai/games into peterz_track_baselines_branch --- baselines/common/atari_wrappers.py | 5 ++++- baselines/common/vec_env/vec_frame_stack.py | 3 --- baselines/logger.py | 3 ++- setup.py | 16 ++++++++++------ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/baselines/common/atari_wrappers.py b/baselines/common/atari_wrappers.py index 6be3582..731ee7e 100644 --- a/baselines/common/atari_wrappers.py +++ b/baselines/common/atari_wrappers.py @@ -213,8 +213,11 @@ class LazyFrames(object): def __getitem__(self, i): return self._force()[i] -def make_atari(env_id): +def make_atari(env_id, timelimit=True): + # XXX(john): remove timelimit argument after gym is upgraded to allow double wrapping env = gym.make(env_id) + if not timelimit: + env = env.env assert 'NoFrameskip' in env.spec.id env = NoopResetEnv(env, noop_max=30) env = MaxAndSkipEnv(env, skip=4) diff --git a/baselines/common/vec_env/vec_frame_stack.py b/baselines/common/vec_env/vec_frame_stack.py index 9185873..1b7a695 100644 --- a/baselines/common/vec_env/vec_frame_stack.py +++ b/baselines/common/vec_env/vec_frame_stack.py @@ -28,6 +28,3 @@ class VecFrameStack(VecEnvWrapper): self.stackedobs[...] = 0 self.stackedobs[..., -obs.shape[-1]:] = obs return self.stackedobs - - def close(self): - self.venv.close() diff --git a/baselines/logger.py b/baselines/logger.py index be38f43..95ae75b 100644 --- a/baselines/logger.py +++ b/baselines/logger.py @@ -106,7 +106,8 @@ class CSVOutputFormat(KVWriter): def writekvs(self, kvs): # Add our current row to the history - extra_keys = kvs.keys() - self.keys + extra_keys = list(kvs.keys() - self.keys) + extra_keys.sort() if extra_keys: self.keys.extend(extra_keys) self.file.seek(0) diff --git a/setup.py b/setup.py index 726c6a3..425a1e8 100644 --- a/setup.py +++ b/setup.py @@ -48,9 +48,13 @@ setup(name='baselines', # ensure there is some tensorflow build with version above 1.4 -try: - from distutils.version import StrictVersion - import tensorflow - assert StrictVersion(re.sub(r'-rc\d+$', '', tensorflow.__version__)) >= StrictVersion('1.4.0') -except ImportError: - assert False, "TensorFlow needed, of version above 1.4" +import pkg_resources +tf_pkg = None +for tf_pkg_name in ['tensorflow', 'tensorflow-gpu']: + try: + tf_pkg = pkg_resources.get_distribution(tf_pkg_name) + except pkg_resources.DistributionNotFound: + pass +assert tf_pkg is not None, 'TensorFlow needed, of version above 1.4' +from distutils.version import StrictVersion +assert StrictVersion(re.sub(r'-rc\d+$', '', tf_pkg.version)) >= StrictVersion('1.4.0') From c5d9c4a1b20bb007f8ff659fc3167fc9147a0d71 Mon Sep 17 00:00:00 2001 From: pzhokhov Date: Mon, 22 Oct 2018 18:36:39 -0700 Subject: [PATCH 03/14] wrap retro envs correctly for other (non-deepq) algorithms (#669) * wrap retro envs correctly for other (non-deepq) algorithms * flake and csh comments * flake and csh comments --- baselines/common/cmd_util.py | 58 ++++++++++++++++++++++++++---------- baselines/run.py | 32 +++++--------------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/baselines/common/cmd_util.py b/baselines/common/cmd_util.py index d69589c..44dafa1 100644 --- a/baselines/common/cmd_util.py +++ b/baselines/common/cmd_util.py @@ -16,30 +16,56 @@ from baselines.common import set_global_seeds from baselines.common.atari_wrappers import make_atari, wrap_deepmind from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv from baselines.common.vec_env.dummy_vec_env import DummyVecEnv -from baselines.common.retro_wrappers import RewardScaler +from baselines.common import retro_wrappers - -def make_vec_env(env_id, env_type, num_env, seed, wrapper_kwargs=None, start_index=0, reward_scale=1.0): +def make_vec_env(env_id, env_type, num_env, seed, wrapper_kwargs=None, start_index=0, reward_scale=1.0, gamestate=None): """ Create a wrapped, monitored SubprocVecEnv for Atari and MuJoCo. """ if wrapper_kwargs is None: wrapper_kwargs = {} mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0 - def make_env(rank): # pylint: disable=C0111 - def _thunk(): - env = make_atari(env_id) if env_type == 'atari' else gym.make(env_id) - env.seed(seed + 10000*mpi_rank + rank if seed is not None else None) - env = Monitor(env, - logger.get_dir() and os.path.join(logger.get_dir(), str(mpi_rank) + '.' + str(rank)), - allow_early_resets=True) + seed = seed + 10000 * mpi_rank if seed is not None else None + def make_thunk(rank): + return lambda: make_env( + env_id=env_id, + env_type=env_type, + subrank = rank, + seed=seed, + reward_scale=reward_scale, + gamestate=gamestate + ) - if env_type == 'atari': return wrap_deepmind(env, **wrapper_kwargs) - elif reward_scale != 1: return RewardScaler(env, reward_scale) - else: return env - return _thunk set_global_seeds(seed) - if num_env > 1: return SubprocVecEnv([make_env(i + start_index) for i in range(num_env)]) - else: return DummyVecEnv([make_env(start_index)]) + if num_env > 1: + return SubprocVecEnv([make_thunk(i + start_index) for i in range(num_env)]) + else: + return DummyVecEnv([make_thunk(start_index)]) + + +def make_env(env_id, env_type, subrank=0, seed=None, reward_scale=1.0, gamestate=None, wrapper_kwargs=None): + mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0 + if env_type == 'atari': + env = make_atari(env_id) + elif env_type == 'retro': + import retro + gamestate = gamestate or retro.State.DEFAULT + env = retro_wrappers.make_retro(game=env_id, max_episode_steps=10000, use_restricted_actions=retro.Actions.DISCRETE, state=gamestate) + else: + env = gym.make(env_id) + + env.seed(seed + subrank if seed is not None else None) + env = Monitor(env, + logger.get_dir() and os.path.join(logger.get_dir(), str(mpi_rank) + '.' + str(subrank)), + allow_early_resets=True) + + if env_type == 'atari': + return wrap_deepmind(env, **wrapper_kwargs) + elif reward_scale != 1: + return retro_wrappers.RewardScaler(env, reward_scale) + else: + return env + + def make_mujoco_env(env_id, seed, reward_scale=1.0): """ diff --git a/baselines/run.py b/baselines/run.py index 8ab71ac..dedca8b 100644 --- a/baselines/run.py +++ b/baselines/run.py @@ -7,13 +7,12 @@ import tensorflow as tf import numpy as np from baselines.common.vec_env.vec_frame_stack import VecFrameStack -from baselines.common.cmd_util import common_arg_parser, parse_unknown_args, make_vec_env +from baselines.common.cmd_util import common_arg_parser, parse_unknown_args, make_vec_env, make_env from baselines.common.tf_util import get_session -from baselines import bench, logger +from baselines import logger from importlib import import_module from baselines.common.vec_env.vec_normalize import VecNormalize -from baselines.common import atari_wrappers, retro_wrappers try: from mpi4py import MPI @@ -87,38 +86,21 @@ def build_env(args): if sys.platform == 'darwin': ncpu //= 2 nenv = args.num_env or ncpu alg = args.alg - rank = MPI.COMM_WORLD.Get_rank() if MPI else 0 seed = args.seed env_type, env_id = get_env_type(args.env) - if env_type == 'atari': + if env_type in {'atari', 'retro'}: if alg == 'acer': env = make_vec_env(env_id, env_type, nenv, seed) elif alg == 'deepq': - env = atari_wrappers.make_atari(env_id) - env.seed(seed) - env = bench.Monitor(env, logger.get_dir()) - env = atari_wrappers.wrap_deepmind(env, frame_stack=True) + env = make_env(env_id, env_type, seed=seed, wrapper_kwargs={'frame_stack': True}) elif alg == 'trpo_mpi': - env = atari_wrappers.make_atari(env_id) - env.seed(seed) - env = bench.Monitor(env, logger.get_dir() and osp.join(logger.get_dir(), str(rank))) - env = atari_wrappers.wrap_deepmind(env) - # TODO check if the second seeding is necessary, and eventually remove - env.seed(seed) + env = make_env(env_id, env_type, seed=seed) else: frame_stack_size = 4 - env = VecFrameStack(make_vec_env(env_id, env_type, nenv, seed), frame_stack_size) - - elif env_type == 'retro': - import retro - gamestate = args.gamestate or retro.State.DEFAULT - env = retro_wrappers.make_retro(game=args.env, state=gamestate, max_episode_steps=10000, - use_restricted_actions=retro.Actions.DISCRETE) - env.seed(args.seed) - env = bench.Monitor(env, logger.get_dir()) - env = retro_wrappers.wrap_deepmind_retro(env) + env = make_vec_env(env_id, env_type, nenv, seed, gamestate=args.gamestate, reward_scale=args.reward_scale) + env = VecFrameStack(env, frame_stack_size) else: config = tf.ConfigProto(allow_soft_placement=True, From c28acb22030f594f94d128bf47b489cc704f593e Mon Sep 17 00:00:00 2001 From: Xingdong Zuo Date: Tue, 23 Oct 2018 04:01:26 +0200 Subject: [PATCH 04/14] [Clean-up]: delete `running_stat` and `filters` as they are replaced by `running_mean_std` and not used anymore (#614) * Delete filters.py * Delete running_stat.py --- baselines/common/filters.py | 98 -------------------------------- baselines/common/running_stat.py | 46 --------------- 2 files changed, 144 deletions(-) delete mode 100644 baselines/common/filters.py delete mode 100644 baselines/common/running_stat.py diff --git a/baselines/common/filters.py b/baselines/common/filters.py deleted file mode 100644 index 5ce019c..0000000 --- a/baselines/common/filters.py +++ /dev/null @@ -1,98 +0,0 @@ -from .running_stat import RunningStat -from collections import deque -import numpy as np - -class Filter(object): - def __call__(self, x, update=True): - raise NotImplementedError - def reset(self): - pass - -class IdentityFilter(Filter): - def __call__(self, x, update=True): - return x - -class CompositionFilter(Filter): - def __init__(self, fs): - self.fs = fs - def __call__(self, x, update=True): - for f in self.fs: - x = f(x) - return x - def output_shape(self, input_space): - out = input_space.shape - for f in self.fs: - out = f.output_shape(out) - return out - -class ZFilter(Filter): - """ - y = (x-mean)/std - using running estimates of mean,std - """ - - def __init__(self, shape, demean=True, destd=True, clip=10.0): - self.demean = demean - self.destd = destd - self.clip = clip - - self.rs = RunningStat(shape) - - def __call__(self, x, update=True): - if update: self.rs.push(x) - if self.demean: - x = x - self.rs.mean - if self.destd: - x = x / (self.rs.std+1e-8) - if self.clip: - x = np.clip(x, -self.clip, self.clip) - return x - def output_shape(self, input_space): - return input_space.shape - -class AddClock(Filter): - def __init__(self): - self.count = 0 - def reset(self): - self.count = 0 - def __call__(self, x, update=True): - return np.append(x, self.count/100.0) - def output_shape(self, input_space): - return (input_space.shape[0]+1,) - -class FlattenFilter(Filter): - def __call__(self, x, update=True): - return x.ravel() - def output_shape(self, input_space): - return (int(np.prod(input_space.shape)),) - -class Ind2OneHotFilter(Filter): - def __init__(self, n): - self.n = n - def __call__(self, x, update=True): - out = np.zeros(self.n) - out[x] = 1 - return out - def output_shape(self, input_space): - return (input_space.n,) - -class DivFilter(Filter): - def __init__(self, divisor): - self.divisor = divisor - def __call__(self, x, update=True): - return x / self.divisor - def output_shape(self, input_space): - return input_space.shape - -class StackFilter(Filter): - def __init__(self, length): - self.stack = deque(maxlen=length) - def reset(self): - self.stack.clear() - def __call__(self, x, update=True): - self.stack.append(x) - while len(self.stack) < self.stack.maxlen: - self.stack.append(x) - return np.concatenate(self.stack, axis=-1) - def output_shape(self, input_space): - return input_space.shape[:-1] + (input_space.shape[-1] * self.stack.maxlen,) diff --git a/baselines/common/running_stat.py b/baselines/common/running_stat.py deleted file mode 100644 index b9aa86c..0000000 --- a/baselines/common/running_stat.py +++ /dev/null @@ -1,46 +0,0 @@ -import numpy as np - -# http://www.johndcook.com/blog/standard_deviation/ -class RunningStat(object): - def __init__(self, shape): - self._n = 0 - self._M = np.zeros(shape) - self._S = np.zeros(shape) - def push(self, x): - x = np.asarray(x) - assert x.shape == self._M.shape - self._n += 1 - if self._n == 1: - self._M[...] = x - else: - oldM = self._M.copy() - self._M[...] = oldM + (x - oldM)/self._n - self._S[...] = self._S + (x - oldM)*(x - self._M) - @property - def n(self): - return self._n - @property - def mean(self): - return self._M - @property - def var(self): - return self._S/(self._n - 1) if self._n > 1 else np.square(self._M) - @property - def std(self): - return np.sqrt(self.var) - @property - def shape(self): - return self._M.shape - -def test_running_stat(): - for shp in ((), (3,), (3,4)): - li = [] - rs = RunningStat(shp) - for _ in range(5): - val = np.random.randn(*shp) - rs.push(val) - li.append(val) - m = np.mean(li, axis=0) - assert np.allclose(rs.mean, m) - v = np.square(m) if (len(li) == 1) else np.var(li, ddof=1, axis=0) - assert np.allclose(rs.var, v) From 8513d73355931d0f9c4cafa0425f7122dfc6482e Mon Sep 17 00:00:00 2001 From: Rishabh Jangir Date: Tue, 23 Oct 2018 04:04:40 +0200 Subject: [PATCH 05/14] HER : new functionality, enables demo based training (#474) * Add, initialize, normalize and sample from a demo buffer * Modify losses and add cloning loss * Add demo file parameter to train.py * Introduce new params in config.py for demo based training * Change logger.warning to logger.warn in rollout.py;bug * Add data generation file for Fetch environments * Update README file --- baselines/her/README.md | 48 ++++++ baselines/her/ddpg.py | 101 +++++++++++- baselines/her/experiment/config.py | 13 ++ .../data_generation/fetch_data_generation.py | 149 ++++++++++++++++++ baselines/her/experiment/train.py | 9 +- data/fetchPickAndPlaceContrast.png | Bin 0 -> 69945 bytes 6 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 baselines/her/experiment/data_generation/fetch_data_generation.py create mode 100644 data/fetchPickAndPlaceContrast.png diff --git a/baselines/her/README.md b/baselines/her/README.md index 6bd02b4..9934c69 100644 --- a/baselines/her/README.md +++ b/baselines/her/README.md @@ -30,3 +30,51 @@ python -m baselines.her.experiment.train --num_cpu 19 This will require a machine with sufficient amount of physical CPU cores. In our experiments, we used [Azure's D15v2 instances](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes), which have 20 physical cores. We only scheduled the experiment on 19 of those to leave some head-room on the system. + + +## Hindsight Experience Replay with Demonstrations +Using pre-recorded demonstrations to Overcome the exploration problem in HER based Reinforcement learning. +For details, please read the [paper](https://arxiv.org/pdf/1709.10089.pdf). + +### Getting started +The first step is to generate the demonstration dataset. This can be done in two ways, either by using a VR system to manipulate the arm using physical VR trackers or the simpler way is to write a script to carry out the respective task. Now some tasks can be complex and thus it would be difficult to write a hardcoded script for that task (eg. Fetch Push), but here our focus is on providing an algorithm that helps the agent to learn from demonstrations, and not on the demonstration generation paradigm itself. Thus the data collection part is left to the reader's choice. + +We provide a script for the Fetch Pick and Place task, to generate demonstrations for the Pick and Place task execute: +```bash +python experiment/data_generation/fetch_data_generation.py +``` +This outputs ```data_fetch_random_100.npz``` file which is our data file. + +#### Configuration +The provided configuration is for training an agent with HER without demonstrations, we need to change a few paramters for the HER algorithm to learn through demonstrations, to do that, set: + +* bc_loss: 1 - whether or not to use the behavior cloning loss as an auxilliary loss +* q_filter: 1 - whether or not a Q value filter should be used on the Actor outputs +* num_demo: 100 - number of expert demo episodes +* demo_batch_size: 128 - number of samples to be used from the demonstrations buffer, per mpi thread +* prm_loss_weight: 0.001 - Weight corresponding to the primary loss +* aux_loss_weight: 0.0078 - Weight corresponding to the auxilliary loss also called the cloning loss + +Apart from these changes the reported results also have the following configurational changes: + +* n_cycles: 20 - per epoch +* batch_size: 1024 - per mpi thread, total batch size +* random_eps: 0.1 - percentage of time a random action is taken +* noise_eps: 0.1 - std of gaussian noise added to not-completely-random actions + +Now training an agent with pre-recorded demonstrations: +```bash +python -m baselines.her.experiment.train --env=FetchPickAndPlace-v0 --n_epochs=1000 --demo_file=/Path/to/demo_file.npz --num_cpu=1 +``` + +This will train a DDPG+HER agent on the `FetchPickAndPlace` environment by using previously generated demonstration data. +To inspect what the agent has learned, use the play script as described above. + +### Results +Training with demonstrations helps overcome the exploration problem and achieves a faster and better convergence. The following graphs contrast the difference between training with and without demonstration data, We report the mean Q values vs Epoch and the Success Rate vs Epoch: + + +
+
+
Training results for Fetch Pick and Place task constrasting between training with and without demonstration data.
+
diff --git a/baselines/her/ddpg.py b/baselines/her/ddpg.py index 92165de..96384da 100644 --- a/baselines/her/ddpg.py +++ b/baselines/her/ddpg.py @@ -6,7 +6,7 @@ from tensorflow.contrib.staging import StagingArea from baselines import logger from baselines.her.util import ( - import_function, store_args, flatten_grads, transitions_in_episode_batch) + import_function, store_args, flatten_grads, transitions_in_episode_batch, convert_episode_to_batch_major) from baselines.her.normalizer import Normalizer from baselines.her.replay_buffer import ReplayBuffer from baselines.common.mpi_adam import MpiAdam @@ -16,13 +16,17 @@ def dims_to_shapes(input_dims): return {key: tuple([val]) if val > 0 else tuple() for key, val in input_dims.items()} +global demoBuffer #buffer for demonstrations + class DDPG(object): @store_args def __init__(self, input_dims, buffer_size, hidden, layers, network_class, polyak, batch_size, Q_lr, pi_lr, norm_eps, norm_clip, max_u, action_l2, clip_obs, scope, T, rollout_batch_size, subtract_goals, relative_goals, clip_pos_returns, clip_return, + bc_loss, q_filter, num_demo, demo_batch_size, prm_loss_weight, aux_loss_weight, sample_transitions, gamma, reuse=False, **kwargs): """Implementation of DDPG that is used in combination with Hindsight Experience Replay (HER). + Added functionality to use demonstrations for training to Overcome exploration problem. Args: input_dims (dict of ints): dimensions for the observation (o), the goal (g), and the @@ -50,6 +54,12 @@ class DDPG(object): sample_transitions (function) function that samples from the replay buffer gamma (float): gamma used for Q learning updates reuse (boolean): whether or not the networks should be reused + bc_loss: whether or not the behavior cloning loss should be used as an auxilliary loss + q_filter: whether or not a filter on the q value update should be used when training with demonstartions + num_demo: Number of episodes in to be used in the demonstration buffer + demo_batch_size: number of samples to be used from the demonstrations buffer, per mpi thread + prm_loss_weight: Weight corresponding to the primary loss + aux_loss_weight: Weight corresponding to the auxilliary loss also called the cloning loss """ if self.clip_return is None: self.clip_return = np.inf @@ -92,6 +102,9 @@ class DDPG(object): buffer_size = (self.buffer_size // self.rollout_batch_size) * self.rollout_batch_size self.buffer = ReplayBuffer(buffer_shapes, buffer_size, self.T, self.sample_transitions) + global demoBuffer + demoBuffer = ReplayBuffer(buffer_shapes, buffer_size, self.T, self.sample_transitions) #initialize the demo buffer; in the same way as the primary data buffer + def _random_action(self, n): return np.random.uniform(low=-self.max_u, high=self.max_u, size=(n, self.dimu)) @@ -138,6 +151,57 @@ class DDPG(object): else: return ret + def initDemoBuffer(self, demoDataFile, update_stats=True): #function that initializes the demo buffer + + demoData = np.load(demoDataFile) #load the demonstration data from data file + info_keys = [key.replace('info_', '') for key in self.input_dims.keys() if key.startswith('info_')] + info_values = [np.empty((self.T, 1, self.input_dims['info_' + key]), np.float32) for key in info_keys] + + for epsd in range(self.num_demo): # we initialize the whole demo buffer at the start of the training + obs, acts, goals, achieved_goals = [], [] ,[] ,[] + i = 0 + for transition in range(self.T): + obs.append([demoData['obs'][epsd ][transition].get('observation')]) + acts.append([demoData['acs'][epsd][transition]]) + goals.append([demoData['obs'][epsd][transition].get('desired_goal')]) + achieved_goals.append([demoData['obs'][epsd][transition].get('achieved_goal')]) + for idx, key in enumerate(info_keys): + info_values[idx][transition, i] = demoData['info'][epsd][transition][key] + + obs.append([demoData['obs'][epsd][self.T].get('observation')]) + achieved_goals.append([demoData['obs'][epsd][self.T].get('achieved_goal')]) + + episode = dict(o=obs, + u=acts, + g=goals, + ag=achieved_goals) + for key, value in zip(info_keys, info_values): + episode['info_{}'.format(key)] = value + + episode = convert_episode_to_batch_major(episode) + global demoBuffer + demoBuffer.store_episode(episode) # create the observation dict and append them into the demonstration buffer + + print("Demo buffer size currently ", demoBuffer.get_current_size()) #print out the demonstration buffer size + + if update_stats: + # add transitions to normalizer to normalize the demo data as well + episode['o_2'] = episode['o'][:, 1:, :] + episode['ag_2'] = episode['ag'][:, 1:, :] + num_normalizing_transitions = transitions_in_episode_batch(episode) + transitions = self.sample_transitions(episode, num_normalizing_transitions) + + o, o_2, g, ag = transitions['o'], transitions['o_2'], transitions['g'], transitions['ag'] + transitions['o'], transitions['g'] = self._preprocess_og(o, ag, g) + # No need to preprocess the o_2 and g_2 since this is only used for stats + + self.o_stats.update(transitions['o']) + self.g_stats.update(transitions['g']) + + self.o_stats.recompute_stats() + self.g_stats.recompute_stats() + episode.clear() + def store_episode(self, episode_batch, update_stats=True): """ episode_batch: array of batch_size x (T or T+1) x dim_key @@ -185,7 +249,18 @@ class DDPG(object): self.pi_adam.update(pi_grad, self.pi_lr) def sample_batch(self): - transitions = self.buffer.sample(self.batch_size) + if self.bc_loss: #use demonstration buffer to sample as well if bc_loss flag is set TRUE + transitions = self.buffer.sample(self.batch_size - self.demo_batch_size) + global demoBuffer + transitionsDemo = demoBuffer.sample(self.demo_batch_size) #sample from the demo buffer + for k, values in transitionsDemo.items(): + rolloutV = transitions[k].tolist() + for v in values: + rolloutV.append(v.tolist()) + transitions[k] = np.array(rolloutV) + else: + transitions = self.buffer.sample(self.batch_size) #otherwise only sample from primary buffer + o, o_2, g = transitions['o'], transitions['o_2'], transitions['g'] ag, ag_2 = transitions['ag'], transitions['ag_2'] transitions['o'], transitions['g'] = self._preprocess_og(o, ag, g) @@ -248,6 +323,9 @@ class DDPG(object): for i, key in enumerate(self.stage_shapes.keys())]) batch_tf['r'] = tf.reshape(batch_tf['r'], [-1, 1]) + #choose only the demo buffer samples + mask = np.concatenate((np.zeros(self.batch_size - self.demo_batch_size), np.ones(self.demo_batch_size)), axis = 0) + # networks with tf.variable_scope('main') as vs: if reuse: @@ -270,6 +348,25 @@ class DDPG(object): clip_range = (-self.clip_return, 0. if self.clip_pos_returns else np.inf) target_tf = tf.clip_by_value(batch_tf['r'] + self.gamma * target_Q_pi_tf, *clip_range) self.Q_loss_tf = tf.reduce_mean(tf.square(tf.stop_gradient(target_tf) - self.main.Q_tf)) + + if self.bc_loss ==1 and self.q_filter == 1 : # train with demonstrations and use bc_loss and q_filter both + maskMain = tf.reshape(tf.boolean_mask(self.main.Q_tf > self.main.Q_pi_tf, mask), [-1]) #where is the demonstrator action better than actor action according to the critic? choose those samples only + #define the cloning loss on the actor's actions only on the samples which adhere to the above masks + self.cloning_loss_tf = tf.reduce_sum(tf.square(tf.boolean_mask(tf.boolean_mask((self.main.pi_tf), mask), maskMain, axis=0) - tf.boolean_mask(tf.boolean_mask((batch_tf['u']), mask), maskMain, axis=0))) + self.pi_loss_tf = -self.prm_loss_weight * tf.reduce_mean(self.main.Q_pi_tf) #primary loss scaled by it's respective weight prm_loss_weight + self.pi_loss_tf += self.prm_loss_weight * self.action_l2 * tf.reduce_mean(tf.square(self.main.pi_tf / self.max_u)) #L2 loss on action values scaled by the same weight prm_loss_weight + self.pi_loss_tf += self.aux_loss_weight * self.cloning_loss_tf #adding the cloning loss to the actor loss as an auxilliary loss scaled by its weight aux_loss_weight + + elif self.bc_loss == 1 and self.q_filter == 0: # train with demonstrations without q_filter + self.cloning_loss_tf = tf.reduce_sum(tf.square(tf.boolean_mask((self.main.pi_tf), mask) - tf.boolean_mask((batch_tf['u']), mask))) + self.pi_loss_tf = -self.prm_loss_weight * tf.reduce_mean(self.main.Q_pi_tf) + self.pi_loss_tf += self.prm_loss_weight * self.action_l2 * tf.reduce_mean(tf.square(self.main.pi_tf / self.max_u)) + self.pi_loss_tf += self.aux_loss_weight * self.cloning_loss_tf + + else: #If not training with demonstrations + self.pi_loss_tf = -tf.reduce_mean(self.main.Q_pi_tf) + self.pi_loss_tf += self.action_l2 * tf.reduce_mean(tf.square(self.main.pi_tf / self.max_u)) + self.pi_loss_tf = -tf.reduce_mean(self.main.Q_pi_tf) self.pi_loss_tf += self.action_l2 * tf.reduce_mean(tf.square(self.main.pi_tf / self.max_u)) Q_grads_tf = tf.gradients(self.Q_loss_tf, self._vars('main/Q')) diff --git a/baselines/her/experiment/config.py b/baselines/her/experiment/config.py index cf29ca5..8cc36e6 100644 --- a/baselines/her/experiment/config.py +++ b/baselines/her/experiment/config.py @@ -44,6 +44,13 @@ DEFAULT_PARAMS = { # normalization 'norm_eps': 0.01, # epsilon used for observation normalization 'norm_clip': 5, # normalized observations are cropped to this values + + 'bc_loss': 0, # whether or not to use the behavior cloning loss as an auxilliary loss + 'q_filter': 0, # whether or not a Q value filter should be used on the Actor outputs + 'num_demo': 100, # number of expert demo episodes + 'demo_batch_size': 128, #number of samples to be used from the demonstrations buffer, per mpi thread 128/1024 or 32/256 + 'prm_loss_weight': 0.001, #Weight corresponding to the primary loss + 'aux_loss_weight': 0.0078, #Weight corresponding to the auxilliary loss also called the cloning loss } @@ -145,6 +152,12 @@ def configure_ddpg(dims, params, reuse=False, use_mpi=True, clip_return=True): 'subtract_goals': simple_goal_subtract, 'sample_transitions': sample_her_transitions, 'gamma': gamma, + 'bc_loss': params['bc_loss'], + 'q_filter': params['q_filter'], + 'num_demo': params['num_demo'], + 'demo_batch_size': params['demo_batch_size'], + 'prm_loss_weight': params['prm_loss_weight'], + 'aux_loss_weight': params['aux_loss_weight'], }) ddpg_params['info'] = { 'env_name': params['env_name'], diff --git a/baselines/her/experiment/data_generation/fetch_data_generation.py b/baselines/her/experiment/data_generation/fetch_data_generation.py new file mode 100644 index 0000000..eecd516 --- /dev/null +++ b/baselines/her/experiment/data_generation/fetch_data_generation.py @@ -0,0 +1,149 @@ +import gym +import time +import random +import numpy as np +import rospy +import roslaunch + +from random import randint +from std_srvs.srv import Empty +from sensor_msgs.msg import JointState +from geometry_msgs.msg import PoseStamped +from geometry_msgs.msg import Pose +from std_msgs.msg import Float64 +from controller_manager_msgs.srv import SwitchController +from gym.utils import seeding + + +"""Data generation for the case of a single block pick and place in Fetch Env""" + +actions = [] +observations = [] +infos = [] + +def main(): + env = gym.make('FetchPickAndPlace-v0') + numItr = 100 + initStateSpace = "random" + env.reset() + print("Reset!") + while len(actions) < numItr: + obs = env.reset() + print("ITERATION NUMBER ", len(actions)) + goToGoal(env, obs) + + + fileName = "data_fetch" + fileName += "_" + initStateSpace + fileName += "_" + str(numItr) + fileName += ".npz" + + np.savez_compressed(fileName, acs=actions, obs=observations, info=infos) # save the file + +def goToGoal(env, lastObs): + + goal = lastObs['desired_goal'] + objectPos = lastObs['observation'][3:6] + gripperPos = lastObs['observation'][:3] + gripperState = lastObs['observation'][9:11] + object_rel_pos = lastObs['observation'][6:9] + episodeAcs = [] + episodeObs = [] + episodeInfo = [] + + object_oriented_goal = object_rel_pos.copy() + object_oriented_goal[2] += 0.03 # first make the gripper go slightly above the object + + timeStep = 0 #count the total number of timesteps + episodeObs.append(lastObs) + + while np.linalg.norm(object_oriented_goal) >= 0.005 and timeStep <= env._max_episode_steps: + env.render() + action = [0, 0, 0, 0] + object_oriented_goal = object_rel_pos.copy() + object_oriented_goal[2] += 0.03 + + for i in range(len(object_oriented_goal)): + action[i] = object_oriented_goal[i]*6 + + action[len(action)-1] = 0.05 #open + + obsDataNew, reward, done, info = env.step(action) + timeStep += 1 + + episodeAcs.append(action) + episodeInfo.append(info) + episodeObs.append(obsDataNew) + + objectPos = obsDataNew['observation'][3:6] + gripperPos = obsDataNew['observation'][:3] + gripperState = obsDataNew['observation'][9:11] + object_rel_pos = obsDataNew['observation'][6:9] + + while np.linalg.norm(object_rel_pos) >= 0.005 and timeStep <= env._max_episode_steps : + env.render() + action = [0, 0, 0, 0] + for i in range(len(object_rel_pos)): + action[i] = object_rel_pos[i]*6 + + action[len(action)-1] = -0.005 + + obsDataNew, reward, done, info = env.step(action) + timeStep += 1 + + episodeAcs.append(action) + episodeInfo.append(info) + episodeObs.append(obsDataNew) + + objectPos = obsDataNew['observation'][3:6] + gripperPos = obsDataNew['observation'][:3] + gripperState = obsDataNew['observation'][9:11] + object_rel_pos = obsDataNew['observation'][6:9] + + + while np.linalg.norm(goal - objectPos) >= 0.01 and timeStep <= env._max_episode_steps : + env.render() + action = [0, 0, 0, 0] + for i in range(len(goal - objectPos)): + action[i] = (goal - objectPos)[i]*6 + + action[len(action)-1] = -0.005 + + obsDataNew, reward, done, info = env.step(action) + timeStep += 1 + + episodeAcs.append(action) + episodeInfo.append(info) + episodeObs.append(obsDataNew) + + objectPos = obsDataNew['observation'][3:6] + gripperPos = obsDataNew['observation'][:3] + gripperState = obsDataNew['observation'][9:11] + object_rel_pos = obsDataNew['observation'][6:9] + + while True: #limit the number of timesteps in the episode to a fixed duration + env.render() + action = [0, 0, 0, 0] + action[len(action)-1] = -0.005 # keep the gripper closed + + obsDataNew, reward, done, info = env.step(action) + timeStep += 1 + + episodeAcs.append(action) + episodeInfo.append(info) + episodeObs.append(obsDataNew) + + objectPos = obsDataNew['observation'][3:6] + gripperPos = obsDataNew['observation'][:3] + gripperState = obsDataNew['observation'][9:11] + object_rel_pos = obsDataNew['observation'][6:9] + + if timeStep >= env._max_episode_steps: break + + actions.append(episodeAcs) + observations.append(episodeObs) + infos.append(episodeInfo) + + +if __name__ == "__main__": + main() diff --git a/baselines/her/experiment/train.py b/baselines/her/experiment/train.py index aeaf1c5..82a11f0 100644 --- a/baselines/her/experiment/train.py +++ b/baselines/her/experiment/train.py @@ -26,7 +26,7 @@ def mpi_average(value): def train(policy, rollout_worker, evaluator, n_epochs, n_test_rollouts, n_cycles, n_batches, policy_save_interval, - save_policies, **kwargs): + save_policies, demo_file, **kwargs): rank = MPI.COMM_WORLD.Get_rank() latest_policy_path = os.path.join(logger.get_dir(), 'policy_latest.pkl') @@ -35,6 +35,8 @@ def train(policy, rollout_worker, evaluator, logger.info("Training...") best_success_rate = -1 + + if policy.bc_loss == 1: policy.initDemoBuffer(demo_file) #initialize demo buffer if training with demonstrations for epoch in range(n_epochs): # train rollout_worker.clear_history() @@ -84,7 +86,7 @@ def train(policy, rollout_worker, evaluator, def launch( env, logdir, n_epochs, num_cpu, seed, replay_strategy, policy_save_interval, clip_return, - override_params={}, save_policies=True + demo_file, override_params={}, save_policies=True ): # Fork for multi-CPU MPI implementation. if num_cpu > 1: @@ -171,7 +173,7 @@ def launch( logdir=logdir, policy=policy, rollout_worker=rollout_worker, evaluator=evaluator, n_epochs=n_epochs, n_test_rollouts=params['n_test_rollouts'], n_cycles=params['n_cycles'], n_batches=params['n_batches'], - policy_save_interval=policy_save_interval, save_policies=save_policies) + policy_save_interval=policy_save_interval, save_policies=save_policies, demo_file=demo_file) @click.command() @@ -183,6 +185,7 @@ def launch( @click.option('--policy_save_interval', type=int, default=5, help='the interval with which policy pickles are saved. If set to 0, only the best and latest policy will be pickled.') @click.option('--replay_strategy', type=click.Choice(['future', 'none']), default='future', help='the HER replay strategy to be used. "future" uses HER, "none" disables HER.') @click.option('--clip_return', type=int, default=1, help='whether or not returns should be clipped') +@click.option('--demo_file', type=str, default = 'PATH/TO/DEMO/DATA/FILE.npz', help='demo data file path') def main(**kwargs): launch(**kwargs) diff --git a/data/fetchPickAndPlaceContrast.png b/data/fetchPickAndPlaceContrast.png new file mode 100644 index 0000000000000000000000000000000000000000..c751881b097ce0200e21fe3150eed2da49919a15 GIT binary patch literal 69945 zcmc$Gby$?q+vcE@ga}Bdf*_L8jevkO(jC&>ol1$M3`hU%l2suL8=HNvhCoy#=C0i3GR|5xQh_!)}ot3SV zmH8J+7h?xUb6XoqRwnR|fzr&$$&Qzq`M-aF$=1P?IVg1u9Ri_*NPZMnc1zxycXdz{AJ~F`1K{KB&y`rtFt21*TtC1R_Y201*ilizqJv1JCN1Qe|-P#^n&EcYaI(5 zP1HY83!Zp1P*YKfrXDV}_y+_8 z*c~mkn2Z}Pw+4bQqLPwk*BrBC5=Pc;^a|5NgINtbG46MFZu1`qh>4{Z8@=6+I|xD| zBPTcOoVIaZy?Rxk*VNdt<3iQZ*~x0qidgHsD?XSakicq;*4^EmRrHIpTvH0Zr2!L` z4|mt)wu=jt{cUX-8pc-hwXxOJ+%z)rc4U^SO*_1CadD@IO^+wzYI^otL$4~WroV-U zV;UG36ljzMJtyNWHxqDhaG0vFBv8OZeB2azY#AB(D^{#An_t{0rL6oiHa51sy}iWk z_{(!r?&e>NT1RJR^#FM7F1#iQSSG&s0M`^URwPkd~ zgw34cXDHI}?c2BS-@mgsb0lWEz&tMx!d3IvPX~oI2ZbJu`r_YDO-~P3SWe;>cXfu7 ziMSr{P}ZgcJ}mUx46vb-h)Oy#j9<{p33WD z+B8S`UE|FT{odXl1Qi|Ku>CtWgHCOX`Dj)`bYy5k|sTxO9=#B;%VrEv>lXIabg7zzjP#W3S@813R{v9<4i(+7| z#*u`9Pf#$$pe^XV%bq&eYmA6Ot*Yl-TwDvDhXMr!1?p8cIp<#ESdLiR+tJJ8r>)qULTr#?zcuV(Xj;1UeL;?SSgYaTA~=@;Nm6#f2&&b;nS&g zGHCWgkV#|@DKj0M=+;tK|L1ww&{bm4hKhxiEuShVlgvH5^uA29a09;e5qt-f{QSua z4W4X{8+|ka;;DSDOgeQgnbptVZlSMjnKDYZfOIW{UlQ=($ zi5+jI`f_?O3RmNWkbe+swnF3b~wZeTEzFPHwiHi_6BvtgUJ9 zFGgxwT9U#;65IL~;J_Ls`UvLc=3ZCJ!7@pl;fIa4a;cmBT#H7lo#Fh~YtXaZ$w=U3 zz@tSZB_;hY7n5E7_4T!PcZcLEWv{HPhzJWKkZ@Y*&sE!7O_c}>k^Zi(jt8y*jgD>| zl^2qYW72uc$hdyA+-78Kj7>=RT`qgIW;10Kz60lDN=}ChFmMO>Oxpjtp|l!RHe{-* zs^=5yjkgD@b%%|Hw-@_iWPGZry5Ni>$oSmAGD}NJmRbYRW8&gSyh?NMNlE*_$>!$d zRHsEQBDIiyhbX)V@BqeJB&TU*#qj?9`vRT1UqoysNA0-sTao8rd$6g*OT4ZfvSgFQ z!KT^S*?q`V9CzOXp-1&OIsZR!UCSc`A*Zb&-pPWT9R2H)b>zrPWDLUQi~U*mjRaGS z6IGN#JUl#7L9ck`#;Ye~GEI%kz3C5b7 zoE$|R0s@~&qz+y{5)2iBB%Ct|Zctud|F>|WVy+R0K>v(bJHvCyS2{ap(k0Tb>k?#d`7|KO%yALY{8{}CEC@V$%u0Uo^Dt{>Rg$hB4e;{Z}!|O{#fuy18a=|lFC13Tu{mRGJR8;m( zP9JHY$DJg$gyqg4xMi%budOMltABEHs~%o;cXx*{=+;l^yM0LFw7%LcZt~0XyS%?# z{EzvB`uC>G&ad+in`-N4PEJpMb{lbXbF+IKf{pkOM(W?#+Y6JSF&xiR zxzW$$G&EOpu(w|Zcd_MEOM~=PRz+`UMnS-j7s|9#X4H+N>%NXh?sH|JTA(frE{hSN zCVvsnWlMg#(Vy@50FtgYEyPO3ISLvZYy2X|f}Pmg&h3q!-lECi(n7sEo9m)CS8?M_ zp0J0mZi?`Wbr8=yJUsl3dHDEN*VfuW6b>V=s;EH9^RPm}A}^3HzFz>lU^G=?U^Q3$ z6P`MpcP9+Y&AYdTGtS-b?(Rzd{#^ly(7DL|V2Xi%_r$UTwTq;@}dCu zhl7K2)&)X2kINn*m3Y_^h*9~c#URz~O_jEp*{X;F_wma+N3@)6^!9{F@=l7-KmqbG zh^n-lEaGiB2ABBs>68BB!~OX$4!0wHu=T^+PU4GH=h6zS?R+hujlCVwPer3K5mT+~#e==fXA%F|~N6kq5v@(0J8xQ)jqhnakD0b0mvX8N#1{RzUzLw zUsdCF{QC3f&!3{(LvU$PF)-E!_zvq0E>9=4Y~Q<|7y+cz74QuA=;DHgkuhAr^P)t* zR<)rGECL4wV4+#GNopr@lV0C#M)P@m96;alH* zj*fu=2Vq{|nr~yJZZ7tvwY5p5I#Xfeb|{AOQ16!D0nD zkp{{1{`Rt~w>Q*v!95s6_4~ER8C|z!q=TkM9~#-jc5v69^T*-Pj+>9~bPX!6TXlgQ zkQ4%sqt)*KWJv?7fn(V5e6@_x?W3ck-vC%5w+tFa3W$RnpzytWFD51iwhut6&+lTr zPW!pM&d1fPW-G%%gvSTziPvRsYERmUE}oz38(eUM>DgW;Uz5nq2I!8?(W9`M{&_Z0`i=X=jP^s`9GysE23s+j|oI4 zm~hwjWYqP&caD*Iqvw0i0iuu7&R7We&IvdP;9 z1AK5IMADW*nH%BOY&e=F>jhFJNMm3{-^0QR*g`S{9cq%n_mY^K?r>~vZ9%A4q6JBX z5?p-E_wkm3x#0{4E$|!2|78HRm!f~C922FHj5gYlf)`leKUW0d)BiUmSm}%M5MyJ! zicd8TH5dd0S`LoVL)-0&kz9#pzN6UIwwmepveM;E<{a7g^Yi<*3WZ^~0)AX{HQGTS z7cj*o)V`~t9)8H8npr6Sc$@SK{1R*S1;PKlD%5qkIx+y3@evsBvuDqiK^U_-OBM}& zZVwU)!09aj>8dw)oMYzX{?DSaqK59t2g&Gv|Nec*7}jP-j9;V=xqx}j|+IEo1f3_RXQvIStG z8x#PE+7rKp{R!F46riV%AR@SVdcv!dr!NQzKRG$E1NVcMBK?g7pbkzuAKfW8dqW1n zcX-39u}iUMdq6Fju4p>dLN)U+lVA<022cxNqt5TxRMjBKz=ObfmzTG+qd6=|5b_=r@~<2 zNl8h9WNO_MpLKkJwRL$ zF>P2L)1*lKSvAJrm02MBa8t z2s>E7M&zz)xFD&P%CdR%kBhK>z5kMlbKsxffgpCd;|0?1lJs~Tf=en%92oz%@BRD| zHCf-zvn#o3?&eOs@X54PX$AVI;<-?o#>^mU=}g9(9p2UId>JwHQ>k774U&&fR2o9b zU2LxSDvV|R+0E2laou+^vpL^#1x_RTfm;7u@NY$BWmZm(F}`}yeowP(?OkCe4}y-3 zjb!LKgqK@gNnJfi)$%SfZph)|ZB>0(G=opg5v}*^N1T-r;nW57SxaZl3D|5;#2nn?Lm@PZkz#vDatNyk}BEgRAN7 z5K}4iisboz;q;=dbpfF)3C@$2QSk02)0E;|4b0oT~X7gIWfZ+YKY1psXN!sYsYl3cu{*C6$ z4AN?+a=8?1QW`8s*o&>6`gU$BJFgYJ+FtW|c~J=Y8oy<>1@yFBc0~Q7nxLj+t_zQU<(q zf!o^uBv)WQqe8|6)9W#*m{{hSadSJzEYKAnMTVxSb%&j3`6rcb&$I#SWo_fj%) zaS>!HhCrdmSlEV6yt!g(mXj9FX6&ya@9`Y19vx(P@H9Cf6H>eRKFK4|Jsj(Ihew+) z^Fq0!To56iv1FGh;b~e8d= z6SY+<9NmwRTT%-HF^OT~j}H&qdnW#OzIro}v8^MNIml*)oBgXW@{d6>k9_Nzl&X|SNRrv+ky$_a+(B)VL`U(YB*5G+TnV8cLHL5o5J$e z#f3Oee{rgetcDl|B%6t%8{LM56aio);;v$Ydn56OlFa>U+PeIMk36EO2RRN)7k;l0 z`H-HJoSYG7w4`DXVcWotT+O6S%sI{95PkG;uFG>_4{o|QA%R4Y3nYRJJOa?5$N3JD z@_9Xs&ou&+HAh3DSQT66buRlo128_!k%#L|iq^KaB>o5a7n(^_M6~MobyANrifJqJ zjb$aJCGQOiF=8!y+$|qHW0Zdp{~^c?)gZrp+WsxFfp(NiulWg{!g=_^M-%opiL8~h zV+_aat-f!&%F?^v7T&O`6)|ROYYj{eFOGZO<4%_*|BFESzyv+jj-!3%{A_|^kgnJG zAT4r85=VGopGPIXO7xWyOXlvV`a5OqcT&a@CyPcgCd1ut0!&|%ugd6>itT*i<*uNr zc^eB1QQT;N_>hxMz_|89mWl_AaX_>N3v&9;8=d9 z%ulDfOf5a-ZElHFtCP1;eW9LPxR&jhT7wEe*^d&@ApaXQSFZEHdsNhS=D%-Jis%nf5eqNh%vC<;mE#AU17+?WvTX( zm4_%(1CjgY{X2a(sJ~H-OhlaZyn(8GkWgQ}D8-~CvjV}+B6>x46!s2ISfF;d^@IO2 z@MkkSpY&2ovi9~r8MokogD5`N130B~cjpDjh3@`-V?av+KHx>>-|b^-Ny%q``hm;= zN&~P6(43sNq-12r$HxXO-;h8x1Go(eh)U$|!`fXYa~aG+9gs`HWhMw! zW4g^!nWUzsS|=w`n)g*vNk4fieSy^;l3x9myZPs5a6`_l6xuD=|3gVgapFUiqIHg* z@4>DD2e#Gvb_MCDTQy%9CMX-*iLuvS{g5Q@MK*bsf=>HzAN0?!KV128a=#*+oSwCg z&cK!*{IBTUu}B7NEf>Od-e{qdENq`r6vR#>4>o-Qt^`@AOmRB@zO*wwBo#EuH=Bw;^&hufp+YbNyhDbNi=-u-Gd{Ebl*$(Lqfj zF(Ot2L?6VzJ5my$;hYO|_@(+Y8x*|?*X@m8OILoLO;I>%0-~pZ-K>xEw=gN|H@5q} zks{96mH3=j(^cyazu$}dCwBtNXJt3vt8>G!wUqafo|Dv+3=+P5$leUS;?cgpYM)9j z^K#vh@Qy-XvfLDna-_t;sgxoqh*3SeIQUv|(iss*uSnp?a*z5fu$#1(U&B_=7ez12 z#rGi3;}^+Da?^d*0{WEI(`E~l=cM|8BU)QuU*6jK1U}maG*MYEkJ}Mtq`-wb?9B6^ zCK%2S0A?);aD6?o^ygiGWN*K_J`D*CZEI`u2gMv`-R5URKQ3C3Iu>zV{%(uR3r`8l z?Yr09Qsa*OQ}B?aIeqROc5#9s!>lErb?%lj7VSi;^87BPJk(dw0{NylYS~~lDBz=s zd8P18d^2D5yzeKN<)`S6io2Sh3754-!hOH`G(Df|t=f*1g{Kh%MuAnO=bG;jLI) z)72aWdr;*CCKhZa&Ldt$+(T|(#if33U!&eiAW3z>w7f`{&MBWXCqZk@F*dUNkkQ-c{^WGTWb*+2cG5Z5im10TE)a zlzvCi1NIPev&x=^`FW;HxBjA!lQWh?VWH9bHpiOEOa07}*6xw7VIvFx3h)y;Fh7s$ z!ssz}p#D%xs-O2VPkdMX50lBjSFCe{U*RLf1oWnVdmXC#eOavc>bu|d8tBa7LmrCn zKiQm8XdfD~RH*jvljtJJ;<2f-HNHsiKrz1w|U zU^c^t%))eK1Ge`cb$-S^>TL*YcO_GuX|syc{-&*n9A}S>O_rCqXgEhv4;hUi13XvMfJ}` z*4%Hy>~`D#4B&$J#`KcK{ zW5DY=9n94XlmgzRGlHyldo&zNih~0ivXHDPecNS}DqfiXC&@El80*}c;BhG~)R*wj zJQ??o**g^6B*$Na@(f5z11?qX^X9G9RDxM$d4fkHagrQ^;*;5c@AMtqry_e64Bb@4 zuZc%0;oJ64=8?{UITE8vHr1s87)zd`}d{nRWQWoJJmXK-+n(~+`a<7u8qzRB>ht((BkxcJ~a?C&7um1CKM zLG|O3_#nD7R;%>3W7{`dVAV9vKHvVN=l#wugvmoq<`+-_WO4^EVu9z#prJt&LUk9+ zPJi`O;UC@{Hp>`g;RVW4N4@kP{On_i5vCC-*05i%;<^(i45b=cD4Jg^K9CM@=cN{u z-HzzG=*{$wE=P{^Q$1;>)F&FNq;RclYS6eGQLNQ1(hF1#4-O|uTB*tyHs3+?jdw)_RX5wOFPPay)O=ON{Sji>8sMHP8dtaa#$-C``@S~y?i&ad0X^K94Q|O zTu6!>Tt|MeK69`remvQJqTSa{KwMQ?&vs@?$4%8b=hNj)Go+&&Z8!x`Je%rHUqXfrm1_CJ;|e7j zgeCTmEyuGiy%2m+7LkkT7H&JYsmbWSsNw2G!Ltj7l~!56ufVxfYb9C4z@%*I1w)m~^3HTsxG$y=FXS z_SYRX7sy^RU922190LGR52oGH(Ms*D`P-21V4LN~oA-+(F$S3>Uu2fZFwnOa*Djmn z@-+W;0w8~i+sh5}ERsvvk?49gK;}m6#1Qcg#fUY*ZA}ZyaR-4tA7VG0E(ZF*s5zP^ znT>gT+}td-3;aP?#gFelTfPEmVzi>NNi};NYN+RsHbHFiNWhIllF5N>V<1kETEtWL zvq|ZKW0&_`;E!i8@;{+J*BSDFKW-$&=XJ|t+oZ|t+fxMv`?DQGUJKM6t7oG` z^{y#f(DVGn^M}Ubrjp-E38j6}9Hf)=@uFY2%^Hv1XO_hArQZK?+adIsWV>IPPPv|l z1CJn|L})pqriTOXF0tgPo+pKe_JMP;yw!k4W4MlNNY>caaa^5%bd=*bv8&qbgZujR z-R06F#sr5~=`I^`XaRaSE`mNf-Me?`B_%H;qX`AQxI^(7U$e81QAP#^B7q!fQuZqn z6=EwlIr0FCHu(r2^P|G1O8x8)x?7w-$s~OZ4Gm<4Qr|)Z`7St2PX~5SHU7ndm0_w` zn)=AkX_(4H`uC(%-QJ<}(#X)LJGwme&ED&hGU)KV9WXojeN%%XG{T(k;5$U<0rF0N zO}My!D>=CiceH!D+zlux|HP?X(&%8ddoGUIDBm)NY8W5+)Xe*$&I{Bo&Hj^7byUe6 zJz8!;$D|?Epeu+U-!;}Q)HyQ;tBw1i$O9z^rj~S@s^+-rP+%EIe)6G=_^Fw#ZDfPD zivRvp=~Gu%*Y^|hEYjqK=M-2q@y@HZKGP#}(CFmF_7w`=@* z%EO-KvR?RT6V#r8Hn`_y*0#V~fwY&^CaNSnnB)5;s~Y8|8DAkyyk)*-#2U858JE-P(vfQ%K32ZZaF4m8mA9RMx)e$Wf)0~&(Q zZ(qVY1U)@cG_tJs14!Q#is(#_Lo$)e&Wpx62azH;H0 zr$dh>5c-Kh23~;ae0AL-I10>@$QNj+tr&v+P7O$lEw>7Jj0L;~Nxxd|Xvw-rzfAvv zuE35?sO&4XEeg!1H@ACRI8F_DjDVmgG^j*a0`zttI&*z<8_C{)n;p}=W!g^3fi*3=((D#xE9k7Az=$Nz!8xLwVWFx4wjQ-~I&K)z4ib^bx z>8t>xNdDrE)n=wI;aFwrFqwd2@f4n6lx-7!VH{x0hrCScqU5Ed(9+VEdm2o zZtuF+9XcZ^20>GbgiiQ{=0r!m(Sy|7(+{RqEaYf4lS@~-4Z7>G2hiPurmQG>Vr@fd za6_JBz&uo?dhxLyAbSi}HC3W2#i?>5*ib{S?_webe;+n4ZwGBIT8BY5LZp{%7BU3t z?2(Z?j-Os$Ufj+*F%Rmxy5#HY;kE&csEp>aXC~Ct%4m3o{#`G&r^CNm=W+hUaJ(L4{ygxGLm$R*5 zxVUF;&kEck=mTbgeqlJlYt-Ne^ku$)y--s54D@Yg0f4my!kt%C85XwPl2Dq@K%p^F zs0D-=zik&A>#sMGXo}MG-&t6{**qD*UBMY!k{Rzz{`X@&Y3p!nc{-4`wUcXi?oaDW zc3w;fAhIwbrt|lTYUK)rbEH`$SmmR1Qlvf1X+XXHGGF>?X)$muW^?Bh^OVG_RTD0o z!+bvr0WJRvpa>&}#7cF*^(^3Nu%h`a%Z2-%kY28EqREHV$}3}*K}DKLYTd|%!GGqg zL(aGt@KH~WY&-4Ax4A1d(2F)=#t}t;BgzU6k_Oa_Gi;yC=W^6fthx&dT*L|=U4Gi) zF2iy{0^&Xsz^rlp0s4$sdNo$#o|j5$V%1+kxqIGB=`$+70Z1DdrgKw`86B8D+gz?iWUi-!|;fF6hK z$_yHp=E1rym*2L4-cr(5?3&6NROeS42>)R=dX?9$#ditib;ExjKh20q#8F27U|KfZ z6|2_DX|b=UjBeYOc;;jg99M{7I1Q3o$%F4p)o@8Fl-*R1ktS|pWJ`Y#5JX-O+oWT^rILkFtptf79XGK{|Gn#`M!6Mxdu;G5%2DiSX}j|44TB%j}+0oSXj zIIr-96lzxd0H8cNHdahQ;YFEAe>&Kf^9{YyFG%@n#h`zW3B)DhVT3JXxk{zw<@Fo2 zsIqYyiPkLU85f9wS$z);e0DDdU34Mp^o2D-Q%Ak+IeyFBTD+gL=vD!JXMatA5%HK< zk|CF{Y(qvgZjai?DQL!CYcd}R?}V1KgB2lE+RXO=LE~AmGcOs?8Zc^A+5tJJcP&tr zv<5trWx52q4FjMhg^TkJ=Ic&Di=P!JTtGb$6d5TGyG!D-H38ISUlLcO&3x_8sHo;J zA~r}ohvmfh;_o4bq*dSbs`jR{;?&)fKZ5c8dB=W$4zvTDnkMWydJT>xIz6j&8A5fx zElU-E@khj}QuIMr7bwmBGZk_Wb;hZhr>-aw?Na7qBq;L9FzfDK`kGF@m%!-*b_e&K zGy@6VfZ5FtJbDaWE-r|6qZiNRyekFJWg#t{PwD|V;)B8Y&bSPWw?gh=wJUP8!PE5s zP_K}gO6y)AL*W6631~5$b`n&h6uD&)^)W4=#-#vy*!a7ynt6-qeixL;K0ju|V1bw! zGaR-nB3F3p!8I<_oSPUaoaFq(ysAy>N&neQ6B9g03Xfx!SdU8nczb8(NgvxF;h#$& z2n35E637O+54f}fN-IAC#G%np=rr($%Q;8+(6P@f%4sTP7jrcCy8zA9Q0bdN2hK}R zS2Aq|*;9s5Nzh4Z-JLenHzW^G2|IQ6CA3v|;0=3yDR9Zk7|Ya=1R%V^>RyBDtGPeuJ;OQWfg^e$MyB zJ}l4V-~r|kU;iV|STAM&gm7MfZ?WV?2G!RuE%1J8Y3_MVpWE$%%)YJ7$Jzh1?vMI4|lF+N`D=+_LG>FB5wbCtET zV|koTK##E6&>ldFF!@-vm-1+k|9824I1k42BzcCF{m&)lf_p2z?PV6@kr!KeQw`34 zD+UroZ$lw3fId?^O(6;fvyMuELj6Ivc-T?!{!9rplQSsT2!Sfa+?>v~;T#`qw>Xon z=2>cz*2otDzvs^Rp};RlyYk z94^Mu_GsSWp!V#e3FZGGdI$!hhb3cq{$s?^YJ>Z7o;mO0Ih((@FXN8Aflk00`~zKjsya(h2e1rG}S<8wh9#8uc)Y= z5)cpsVGy;_7uSEu)2hxTmEp^KoYpn&%1B%NSNwKKMEG?>gXf9a7B}t$z|_Jo`jTWU z22OiascC3}($il9Y0A?NAcF#})+^8&IoqF&1D@L0y?P`=N1vYqTJj}TF7m(HKfuz$~G3973#48z$K_nk2ke<4{ zYq8&_u!vA7cm>~@fQbRX?!%>c*w_flvj493s}@0I!%y|TAS!OSAO^9I(;hQ$L(!7H zMj`5-UbiO%)d}r)q4&J?6?|uft2s^hkW1?8NiBEtQhoz*X}W6uIh%{9h{z{-c`oDg zfc%o9D=zJ~pN5*A+a0m#7{h;SnZ~42zz+zS?|>c5YY|l~8});^61p;lx^AF2=Hbgk%yHjEGH38Fu98!0=)~1sqb@yp)W7n?Iy)Xxvii z{p`;BH6}!!z%uCatRd@f?>KQe?=uqVH_?cAv6=;p{hW6-Hhq=>t58(GMa?d{a#a62Vn$CnI+|QRv<*?#QI6(|+4ZrB0P$K=_-`v{~bT*eb4~yt9 z)>DGGJt!iaHWnwc>>L_LDl$>MYWO}$u`BEs*Tyk9n+zl|s4s|!;Nn2o8G)`_3t|Om zEyb9WZjSgwwXiIH<<{cSnLGha0YH>OxbybhDc4eVH5!XDtwbP}4lRW4KN#_^btR_Efwx1;$Pc}|F{HVe8bfgE>$dQQ#N=b#5 zrwf(o$qHmD;SS3c5yMcq*$(pZ#B%YNx;PyYl9&0m3u!StQ^dL77>>L*Ruxt48@k4b z$#SPNuRNx(yl7eBaC=BW^$4iWC>o1SHm~0+ zPoAg>z6&=2WgjqNph1Wfgd_ty_IA%MR)*VuOAKY;#N9yr+R5X#TaA79}M zqkU5s5RYCnrJ?m=YqeRl%^xhvFv|Bh$jn4=gbcKOioP~5v1^h^brjBiHmY=Gnn18} z&$pEAt9BUF0gVE{f}~BT>W*^!?Xh_1axQbI$@&X7=hL5R%ES6coBI||o109_;%nWQ z>89V@7Su+@Z1+BLsYL2@82-(m+yafjhLbVz`@|!H-POu;sFP8Q5rV#{L1=hje}#%B z@IV&yXWM}?I`vg@9@!AnmkvaaM2q3GR7rx@ZJmnUxEZ`SV-ufKRaMgvnp;!@wVCoA z2QHpU7Tn#FGBe)=kNb9l)!XB_erfExz_vT*^#ejOoLz{nDX(M~twhsUlMeuvRt#2z z<2KxYe|tUlCLXR_VH{i)F?fAH^N~Yy1J0Mnxu(3bzAamCErtMEM9X(>v~%@l@jKig z-+4D&#|)pTQEW+FudnNi+$95=qF~A=Yxu| zdg1~*7Rmnu{li=4jc~BH8VUMkojoF{oF$=H4jS_db@a~Yc{dnf;Z9hrcl^(+R{!bT zKOvke#DpF`H*NhF!96trv#S>3g#D0WyzaoOYJW0jGW+x`upl0yHwibXvpiSJmXld> z;p`{5SE-bJNoWM958$C-&ZMKtnhZ`lQ&?7a)CVxoUYo@TK2Pk-&m9Ggea&_6Zk8 z2`r`GM26>V6B18#1Uq(L$K+T1F~U0?Epme_+#l#EmPwhd+VsxKUsxy}nHK@33K7_# zG|-rgJl;uNcHiV)R9w$ijX&OPmlkWVuX%envej_xGLcCM^>kk85jSMTLqMHj=Xibl z@g0i1F^iAe3+p+{AmdDvNkl(3@&maN+qT`_i0#RU9J!QFIBqplTV1COmsYktWr4#6g2Xj>>Gc4mEunFM=Ar^&0vPJtnK~mfBfJ z-b3w^87z@^D=Ug;d*SNOcv=|s@%{x{vRvQl{(A6FFs!hG7R?qaIarAH7dX&YYvRRl zAktjkw+M|)Dr_-&j%7-6$G=~yJbs_-P+4Wga+S|GwwO@j;>`Do^`s}!sj-H2mb@8E z^?>QK4>mSTWPC1bSI4U{G0F|O&nW~!W#Jhwoy!8FW_jsroEDZEe%FEo6c=FB4=s=~ru6oQHMMEEu0bf2v94JoeWK z76OU}R1-|#0oQW^q5TVT@**L8An25!@HT3lNhlVIb^(mtzbJqdwfxloFSE#Qhla$5URhL5}F(~xIDBiGvm(3a{BXN(wi%6 z<*(E)uD?#9z?B{+o93Mx4k~s?w>oK=&uAH|0qI;sG@lPLN&oCT1cI;-8_Y33U_MZqq}-!cPeLtwYJKe}n-1{TI-iRAYZ@Q9=D!ElJ*n#%9!&1>{d0!+d#foHaLf6s zg|C^IBHG(O`rK^ODya<@NdM*!!Otk&k1oPIN)%f^0RCRe#b~@gXMpV2lO=Mf7MCc?$_ev8 zY$8srq7{Xrp#CWo<GqBC`V%VPLHvzS-Bv%0eN8?HaO67f&yNBCDy|}+V z*XslugQ2qv`ZF$2+=W zquH+?Av4v}CwBta{MQS5Kwj#3`@Yr|vLnU#c5t>FSEUq`gM}P~Kd+!$rp* zgF?j-#_DCVBo_+#_ip<}v>vybY-Qf4dSEct#3XCj5L90`90*AkpqQHjLkD1h;Nmr(@zTK=6?Pi$vNE9PU@9rt~H86rv|5Q>ph{E5QJbF&?%{;*xYY8yP zUkeKvL8)KpXay!0Vt)Pl3#LwJVl!3qL&C$?6HHUye>$8jrT~m2o5ui{dk_NyoC#nA zkk4hWxuYX+I75OBkV-FqWJOZ_>@`NFPts%l(K~vB7cO%cIoO~;yI<+|^=$`a-g0E@ zf}luf_#$gubJllMk07KmN{Lnn`7HeCi>=f6lMC7v$L~91M>y)3pDoMR@Kl$!za+-K z&6~X+S!e9$PcGc{sM)m9pX9{Y({!E8Z03A^=SidF8}U9G5r9&+I-=CMDr?_w!bfw= zt)XcGvz0#rQ*8fC+&pk~f=Uohp@OLq5&=%LA<6~!O&CpFY+PKhYJSJ~_&DW!e}Cb^ zIU*Da<*{3WxXwEhzy~}+LPGiy*igd7+hXXH61}X!G(?8yzAp3g=a`YE-Jb?o+yvfO zJbBRNx}1^VQZ(FWE+>WGUE+)$h)J6t^8h4{jgEG}gy`B-@_W0QdPA^o^ z{!0rW>*LcnY=~LI`T|OwyRak^O$`K)*N483p(1JUpQjG%N;dnvrs#6` zC0Zb<79Ty%C>nyDV~N$7ZEZd*u*H&=56{2OjjcMA>zEFyzU+<#avX;ie|@bh=;;or zROQ$-E9Yg>Ubv=8Z=nC>U^6}VbEE!|1ieoy5g-CkhVRW*MS_{<382D!^qDEoo$c45 z0KDu=G$fGikhghwc&clN^sol!=gBe^&5x?9K%WJS?}C}M&$)?}&no0-(?%=ieAqM# zpND5Xa2c3Lj#K1=7CA4E>&I*)vclEKi~irG7X+FSW}c%KvipyR>w!^>j4#6|$Psa5 zyi4fctP_)3DVwqL+U!pMQhHVcA?1a2*B?G~OfN)dAMA~-F>lAQTK>|Z%O6ry(Eb{p zc~Z;24shFFG)zua{O}C{XjJ~;GRB#`)!C(fS`;q+l8lu|zoP#5=9TBnDE*jdQ@m5A zxP-QYAi#7f@%!O&87B@8Y4fw?iHmh~6@1m5T12n;+&C25@tSpzHw zsttpVA9#SrFX}eVResmn+6ty8i-m8%j%`i!U-?we*SXTt(LI9|Z4e|!Mfug$c|!f0 zCMfIb^Y+{EtpO?ui%9AH(goPRx38b*lq#rto2&g>=bjq7D?Au!3D1_8WKj2_p-a6l zCA!0fsoG9U5{Xe~P0FN84>EFaKeW<%URgL=HEeDYsJ-xhg`$Ituk~Qz$iVD#gGIZ# z8(~9Ny|(X5zZ>I}VOeSx;WAs=Rye zz#w=`12c4=q@-5BkQ)3clBTeAiRc6P6ZeyD+9Z_)TzUQ$Fct6&jL?I;3ZFLvsf66? zNEnhOpBj;!O$!(hq@}cA@-OQ>v3)SUI007~-n^D~L#vKV`Na&mHYcRJYJ*GJ9G9beq^p!xWg2Q)a`3#Vnk95{nkW!UxU zCO{a3U^v%(ue2+k+kWM|w&T(Ca9}mP-~uJW*CfcB=zXz2-#-abpNxV|2TejVJG1j@=6##07<37eN)&CRH930cBYE_cWC z6E=BH`x2=5y*3#Zl@3|QHO$`VU}c4FA-qIfMo@Zxe7vfYw4&6HHo?mDTovPRI++Zm z=yR#O&B}_9Tdgg+cSQ45Ie+7$Xq>QcM(ovDUDJWb{3CgF=V)fG`ow;HbAODskARNP zgOHHk#@x7*zeGv7e?vm}qTsD9;(vtgi72w6qaN8e?=fw2GA)jr2V7TAs}fHX z3`89~VC>BW%zgV}QY4rz)ad-Ea^rd?#uQvNNtUnT9Vt$G?9_x$owaGtmbtS(eIWDs z^qyjJ%ANhWWHjnops+CrO9UEj?_gR@J_6wM;qUGso|CZsiJqZpxmq*imA>!` z;-&&^4o9I!X!;k%r@YrEbAG=qdb5nT71OS2CT%RG(H)WGR4|N9)^oe*-{$HB{twpP zDy+)AZTlS-Fa-rgLO>8HY3WV_2~p|p?(R}bX=$WHO1h;>q)S@5ySoS9HRkiY>-gTS zwXxRDJUphsJ?`<(^E$8J`60LFNq&pr?t0dEnTxuzrRKpQmiPRvwfnh@_MUZDt~*54 zAuX%DQMLY}OsQ{5s;+ddltwZlji`{W4bS||uI_SZI?=MtLp_Ozl&s?Pn!te4G6D@1fvPN>sbv zK?Oyqrh&!#bYD^pCeE5OCwgz))U&b%De|VKtc$v;og7(`>TjLZL`3f{3APqxUH8Ut z^Rfl7tG*RTHO~}0nO5n#%E@c}b!jv^?A|^>Fe-w1@QJo`&cp%P46BF_#5& z5-KfLdU+x8HS4DuI9LT%Z|U4Be2kGnK~WvncJ;7FfX_|vvqp;Fd6{Y1{lGff>E?E2 zXV2#m1^lE^p$-~urOI5|%xq$BbX=T0s`2plX3NaCx{A#``vwP+KB!-;;H?Em`F0>%ogXk%BYhP8S2-ScYAm&baeWNE%4SJ|==^Q#{tGV8NY0^5pMQ3&e;h97 z(X#m_jP{m^rAE;)hTN0#Z%6Zrj*d>tRIDWt{J=$P!US8}uTh@PGS|HHZ@i>-Ew(TFYxrU(kDI~!XOGb9#$cYuPj|ci&2QSJq zZl3X4!9x?}{XW3+$JpiCu#Z315tCMvN_p{-eDdqmh+#fFJqg>r^Emd1;BZtC3+|L4 zwl=k7xb-$e&j;8x#DLQgq*UGVHk?vs&%W~e#J@Bf3c2rnd%H^mKWykS6?gXw=M8L_ zR+W9vbhAk~wk`NYK2*6SO{LL{&n>^N=JCRdqg`OPE@#6)3dazoQ*V)YY>iK$`>ILD z8!(j`FNwM+TvMDYA!MuIcqDPWaf|RzdyJ;^RE8lR^T)5;{aNXJYdTGM4S2X*_4_t7 z7XneG2e!Z3+wr653i@cC;0JZWOasAqAF7Z$Z4{x;tY(BkHGs|(?d_@B2LuEt9DuH3 z(fdx?p7d`oxJ1U9S~f}3L0BW2TJ6k|NApBzGpbd_B}l3`q0P^YnB%`Op~wZk(Pp@d%@_QjhBTH|jPT z&K?XX?9-pWx#P|sDBg?9Cz%cn{-cOKc^%XKiyzE*l1&C4*Jdle-a7kI)}evsmKo>P z$K?{&RP7(w@Yud`M9^(>VQX1rG&HB!Xwk&9@8mzRoJ~!P-`W4H%KcY)X4#_m)O7Xc z*_6PekjHY(dN+iqBHTVO7dOmb>S-IM=gGzFLdGu>xpe~dD`1p zE_D@GLdq9OMSIV@t4_YeT2zRZe|DWb2#C5BuWv$h5tS8K>(zOYYG}J*-etVG9aPdE zaKmtZ^}&XQ;RpdKDfJ`7?PF%i#5-Pb6IiOY@vf2JE-@nP<6=G87HML*3y&|H6J<1~ zZCJSTU6S7M65m+pQN8ibS5ouZwOqxnm&Tv54ywH?c!-$Iqtksq(o~(UCu4uEI6rdu z`0i1}9fYRaZnJCj>0T!@@i*!W+7fnzUE#ao)pA)N^C-%^672kW=^ImW$C7^(&G({_ z=u9&EPvA&qEq3o6j9ktzR|nxRK}7D#W(@D-amRFaY&{mHC>nZAZcJg4UM2%(Y?TmN6FdUWOrj9VOlr2HvXc>hX9AF(% zvni?@VhY`y6*+FW<%Lkmiyj;tbR}?mO|?WNn>)1FVUm5oVPk$%f@KktF`RYJ>kJbyAmMmqRgBx1@OY30MvPbx;rmt90?JkV zRewJ}xWix9mFuKjz#~{83w-eG;~%Fyn1?IGbRn;3A|l@(T5UU?>#qLoe{@{` zY;20bHoJwG6^j|0I1+l_lW0baVEIvv0!K9UP)G9s7};>NLTl*zLFsq+!&$H*f`!)X zjMfkKsKPSN<(%4>gJszG`_kDFUCa#H+n-g@3ThL-I%+1TQ`31!zcqwGBI*wBu@Jxz zE%_hel)VYtc}{|a7>-X%N|k`sdFipXUNQpFqyA=IMU-;{x5U(K5*8PQ?N8UWT#XUR zeZgrYRi%61UZm>xjHRLlN+agNZQd>ZSl6$c%unSy2+bKodxPUsr3=Tim%hHfD5vP( zze)gqIB69^egM>3QO=OjV$>00ES8u#zV{wOucwxy0uDVx2(Y;~S^O{K15Pc;gG>acp9fgA?erB}w!oKF=pz-$K_+6FHX zdV``XGmF07WOvE12^Bo`oRn{^n`t%13ml&@mMnzTYJd6#;H)kHL~y1Hbzl_l1{{v7 z%Lt3H($D7OKl{?eadqX(-ghz&-}3rtp!4aaQ-%AfELMB=b$BVdyTtce@sfZZ;krIQ+t~I8zeuqXG*ogUX+oyEHh3CE8((2Os7hyPV+Z2}))}N#C#;=~N zsrWTo7nSEd9u$fgd%glLS8xf8%>0Gb`BQMP{@m}+C{G(ay0)ol=T{EXAi2(#0yB)i zDhG($*ev!%etJ<`jv`U1)Ya@)YPvh;aW@f>oJ&~2JDKLT1XVwev|el@ylpN?LFF)k<8x*pBj8nT zIh_FyjMnObJVhacP}Dwz`Bre1R2#iI)&HU%c z_64QyhwEhe!#%8dFxE-WnYX?iHaR=xxlWj*+!;Ny7%D~c6XTmbWIomHKjIDxV}Pjo9F z_yerHdBVrRsvFmz@ON2IFB9=IcM2Jrq4!d~lly+_*~(;2PohYUPrjq9D!GJY@31?k z=Udk?cH5S|o{wAPek0MH`s|177cxJ>Ix);=vpu)A^q$w>jf!Z^u^F6@ZqKv3ut37% zT5(`o$6ZJzN&WKE%v&6yF!5Vnf`Wo-HBRrZ07Nwp7Aa`!zSFU|ysT@gT8*a%2W&k6 zbfBPXz_CL0cfhq2%I%V61${>BsK#CZu*G+|H4T2WD-GYuSl5>IY*)LLQuIgSe$hY5 zQ8HNxb!HW+?^G171vhT8tAE84!sff+H<_07T$7=*WZ-G27ZiH+p~~mi_xtVzT1uFT z8A6Xb=D40%S^v=K>wF~OnlHbA?^dj6%lp$V)wkeAM3PxxoM?>rjFp^Dda^I+2Qy+3!BsA^Q4#f^ld5 zm~HorvUR(z|Dh1Js|BUL?UBh>kxCzzE=xt(W-67-xzMpR5$9}6#Pnw9kkE+4ees8< z36UsqIU8nh3$!B&uvvR%-POpzUj^CR~vQ3?coysK$Yl<p?YN_j?L2uF>FW^&>x$Rs#I$E3#hXyln^DKk1oV9B& zPlrboj7C7vZruE6I(wgEvG#Ef=iS($ z`HU^>lC?3-AD)gs-J6r@{`#1GJ?FOC3*U;Z6qOE3rsx}C<0grEF^(XpD(BF+o#lQrBm-g zl#%h`w%y1c;+o)HwIJDhF_GRcx;>8S|Kg^4PE4py_^*Xm923USmes?U1e_m=C09|w zT?_<@;oOG~4t`naz;{zBF?f^bg6NbhD^ru7ma?_AMG^8a@fjLMM=OW7Wxte^nAduo z%hTQQw+e``Z`WUBk~5(#%s{5Ubu z!YidP;=U34rZ2<}tvg1?OOZyU3k3MO#`z8T*9iP54WqH23r$fVXfnTMnh+g_IwVwo zm3A{ZRjZ*dFX<>C=))pdcFi zhW5q|r=3M!ylpJ(Xes-6GF=B|ArbEL9gQcCagIsnS$&VVvJiz`lOOp2!USav*_?F{ zS5w|SeJ#mGa$n-@+gA8?WllkFyGKUW3m~Y&)>4(pnku-F?L$2WD0Yge$31OrIpw6n z>HS|fHmJl^FKDxo@@;rIw zq))o;#C3FEh{jY#?$!f5Q*N6h{G2P|mmbEwqs3VQ#Aeo_ZmXgJT0&x7`uFMNH~rFS zLXA4NQmySQ*W2e94QcatN|wzG->KJ6QntR+>Lo6=P)GxnsRncs5I4Zo(PUIJt+N=lz5f9!p@ z!mWgo@!0qvFV7X}V4tw{)*D15dz{e%N$#wff+KxM;w)a<{f|*8!2Pap407x}wKfTh-)ik_)N_`?;;V);nK>sqTY1)_elzQ?kPEu_{Ztqi6v)dbYu|I z+c&lXEUH&iF_N+^ZF*JO7Q5%~g?`tzR!F4))lD-?QN!66_P*8fNYx4}a;S-%Rr&x< z#Q>b42qhJj`wt%I?IZ)KZ+JNR+qcII3_;0~xxAww| zSL6-JVQUm8K3Xrm$&dI7P}Eea?pJo;Rh2!!H{y1AKNK)k)z9-c(I8#Gtv;dR&Nngz zX4#*zvZHgvo^PoX@V{e=;sm|C>6MOBE2mu685IgNx`7eJ?Yu_`9NCkTmAX?JYHEe* z$G;;N@K;?Bn_Z^$<`Iu$!w+&rd!mlKe9M0DM>TaeTY0D_Kij4&uoV&yF_lnHo88Nk zVpn1Oyz2Y_Z~bF8^_J%qriv3iHpzya=BsmuoyujMkoxD;5q-767Q0d|4&YX{l^^+7}*z=PcTHA?rGN35em4lgJUp%Z$eSi-F{@m1E9Kx@6&l>)QJ#u zjG#F1|GlfYa_MRy{8(|m*eb~6a>lddc({X(uq6j_)pYf#^s~oXoHLW%DV|wk)qQ_Q4w4t9NV|A z*}IwKgA=F=+dp3x|KpfS9LL|ns>c}e!0W=kmzPGBS>bl~+hhLM-}5s|=FaOY62At% zqA$v{^BjX!eBx2wZoO_yAZIt$fM6w`GVb8-dy=-d;PCV(+t}HweEyGpkA}Ez^YQZN zr6EbO$1%aJweWiR>0Xad{X$dD3ux<(tKk#lDse-j-*Mj>5`%)Oxu{@e-#w7-CVE~# zW3lSUW=!yIsz<*haU8u$TUTFW+y4H|6c_5r8-&Efolt`S4PP`*fj9vWk|<>7YYx+d z+Y+}AtG$4H>h#Wt>T@8M9K-1m38Gxa=ARw=!R7k(>Gvb&i<>5ra;r6^?=My4ZU;a$ z|H>_20FJOr2DklQo!GME^IMQY3n`v58 zHJnoVciOj(_X9sG0~>otxx%yWA`uJ{)6)+Gu8eRj3xx7Lx3-Pn+FDXEx;^4{V)}5> zZ@xu$w;c1M770A72Qc`sfWr#CFNMSBNWt9ud*O$n>bc0N3n~#KtWuSBt>d6SGoK$OpQg8dJlzq>+;J#tZmL}R zCm&F%WYm1<�(;4U^p!ZhDqkv8l!+HA-mvRtGD6poy&m@emD@k92e5_sm7V_&v0y ztZP$u(QMC6eo2vMIvSnco}CrQAzKshV0n3UBI-LkpF8p7{(yb0CQ)AWembUMCU^kU`i2 zBucqQRaq-kWnF&m8;oxk`TVp>k)+2~Fc<30>dd-uSmV6pJ(*e5=?wl&;%%x{m*@-P zJE)EM+tmcZg!nt z5Q3aBk6mpW0c zWD0SB=X>KR=uit>q_P!gd0jH){Twcso2`%@>2k|LaU$U3tdriDCZr1 z0C=_jl|u-a)QrNj?gyp6mVKe|c`Gy>1L5np2h!nLD-9F1;%$U1D`sIi=dyv#-N1SQ z$1bzw#kqq-F!}fFzBpQ|;HZt? z5SxqjPQeR{%(eURu0+oAS!&etLo_OVw10!&)(6bn9|IunKxY~zhYK- zPM%0%W>_FkB|HIxxw+QL|9Ao5X?(}B2oV9K!YSl0)O){G{35AI3vv@2bNkxqxRFPk z{MB%$=6M+LT?hOPp+}mVWty5c4KtOQUtAHLrI=RwQ<#rfxFY74-wQI2b~d*>CbUp| zLn?~WS#|oa%>WMP1SE?}>-ksdiNXkvi(N5_!E9?8JcX5Or4Isiea;+@BuvtE9nm}; zT{b@}*kSIbRlMkoQdK@9t#dV=n{sma`Oi+)1vFCq;9Ablh)7cZR2$4w+bmcCjkIwv z_5o3=2X{Za_2FQEGy(VPrM5OX@X%WZ1|k6VMO+;a9FSd#{zuBfFVA^?h4CIKLpbF< zcM9*c1Gj_?Mjls=J6<@RG~X?MZ0(v@y;tea3#PG&!;-|jwOu>xE6FAAc5xWzTKBVv z2LEWkaCy4Gk;{R@=apW6n)OYzpL%P9T*2yiv*h~4*Jhx7$!Rq`CKizG!1#w|rKqql z96-$hEeH$l{u~Z?#8JiMT4IvuEPZM}WWy^gCeKU?~&@Owh;9{!2~t%#MR5DhO{rS+N(orEf8@qPfEdcEl<|JMu*7$4tB z!dz>CGy(TBy9JxcmQ|345MY*i28~gu7HBva1*vtmP*7=dku1~MAa2`lypXyLmC5`) zwQ2-`=tUy9eqZ#gfp1SxFdg$Tn!|3>a0eDK{XU)lo6aCI{9k20W^R*aqg7XQnar6i z*_xPDWxJH*90h${SGYHI>1b&iyK=2B#A7J`34S$Tv_>JX#iSu{bX0NjJ}@*bt6XailG#>?Vqm9y<6azN;+2E9_Pac7o-kk8>e^4M&9Y7&wdeB-(1 z3xVGF++SKM@r+tpy6UbkEXB|ydlR2oLqu=EL`S!vP_JELV>60Dvo1C^HV|Hgn|u5- z*CVx$-uB{yy~#qYUpeSuVZ0s^5&}(({=4%u+UAV+zjNJ$nd$BAUFmVjd+**o)bNI0 zyqSdspSZY(s0)LLh)7GmPO7!w=Jm?vs;Fq_8Ge-?yz@=Gmx@QHaZi&d!YXi&q{ZQe7`$AGpCUP)o>rK!$+26*vTLf@|@+ zOqNoS&OaarFaeQ*tD*87UYK=5EO?&tpaQH9lZNfr7d^3^JXL{m&N|NhMZ zoN-){F3mH;o^d^cY^ACL+-RmtA2@K0R5ALFv5g{j(qlB{j?C46Uc%ga1_>L~(g_$| zVCMjcF1x`YQ53HDV#LaL)z3OfagiT9;^8MQ(|9s$_Sdx7n5mjv*eTNDBHB!82w%cW zD-q%ot*c5CnI38B4|+@OP+*Tkh=bFXQxJ0Fn>en}Ps`~J5Rt+<@u0pDMdmr-4@eCJ zq(scmG)|A!Un%*tTTZ$u$BQb`^l5 zw6X?|BieWN_h~qf=v2!+@~{!;*i6aNi}UlHrN(Rz`HrZ7VO>;2oBWZ*pv#HZktx|T zIU%9Twzu~6KnC)xS7`|limb&BDJ&kX^X8e=aVzxbu?~2KyM?DxEXiW$9hq}elLQ-TYCeG>R z&6JGm*RS76s;;Q`E~5{6^h>amI?a7XqAi1vv%9-kZ<~~)0S%bFLff;`SHqj(Kn;Q~ zHq&<53ch@&vsG;$e@6=b8b0X1Xa)$w6rSyj{_z!UT1CEsVUPs;by}zNI=EtP3?4d!2V=i(WAl za3_lfTYTynYB_|eaEc4~ChRI$6Jy>}Ty|zYBapO0n~3U-aUQryLs|4*aZH;rCz;J5*v%jkspz)DGL)Hwd{by zw{vUqD%Ekj4jjw2muDQXG!CAqh)%L*A$FiaIRjd5;bZLqUwFDK7Xx9Bb%elIZ4h9u zyWG6}dG)kr&xxMIY~6tO_b9#b0^Zt^g8S(J)dj!Zz(26FKO8D}nOFlyBMDMH>zavc^9SEueYfwWuh+w)SRGxgzQ}fZHq{k#WWffL{2z zeYBgB7W&f(>#mThQ%<@>-`0cKxVV-fhe7{u;XSo_(oO=+-0J1?>>=M=m9ii~_p|xU z&F2yltgtQ~h%B%4F|&AFbTVjMumH`vwWH%}Ma6mow{_6!fZ<4Sv(LG<%rz}3DXFN? z^$wVbK!X;5hW49iXsnJ~YVIa+eCCk(1HBejiye3fR5B5ZE*jJhbM{>>AH}ViUFwQS zrh~f69toA%Bhva?4X&CAT1#x&(29c0Gfk@g+Oo`dr~YtSYS?^WI4iwcDUKe){xx+( zkx?J(6AH!sS^w=Xaaq4cN1;FQ?%5zchrXduvbsR9`%NJJhg>!2`dcv3@N{Q*JiECmd4&ue%rEjer;Y$Xrvnv{H<*jq zz=#GTh?@x1L;J5x7ueKq$h_|d0hyM?)u{ZGX@pDznL%^$91de!qev>3q zW)6-_CLzD1xHu9T8dZl^o6DUmc?gEJLr4bVruK=uAyc&re`-l_ zbKNf~Dle*77Y&B8)Tfu?7bGDm{{z?ax z7BO=QW2YB7j(J=To`Nh`ATO{ktBn24cLM6BLjn?*84P^&pSp!Ksn=Xw(rqmZ_|UxI zMlf4pHIYcSon*aaLzm;YvUHuN{?Ge*zE;m84KQ_pNF&}pxLKpC&6F8c|5yDLSoa5G zkr=T5ML51e56;7~_@$>)?_-J=YJ_QD1LSoOxWOWR< zFTtCsF~6QIM+ZqTOiX^yb@TJ{(=sxKSBL@$m{A)k5DrF8+^+#>VBH=mr;@^i{w*|> zMq1j_k~H?6tAu_NR;6j_{VP7VrkTHO9v==?S337Bdu?`K`5{k|gyspys&CzOSj=%K zu^IYJrEi;A450uWKc3YHJlZ+3`zFbkTy=$GpPMqK@dHTs^V!Ddp8MS+hiVc25wsT< z@p1@m@abI6JMS;`0jNOmV&}OxYeCK4ZHP6P{ad74MjBAdhVov&Y8-`z>7IP`>JxBx zm?qZpQ&1F4xd7S4bgm@gniZ#`Xu8SY2_@{3;-lUkCgIeqE9ZBfuZXwY8DrXwCI|1S z+VY@O7r3QBt=XWHS+T91dFk|2sIROHopX38Xl>*pE`jWyDSghJij=&6%(u+$3q8Lx zdGwLI57|OBHeF#dfNa@{gyM4H_gdFuq5}*Cb_mKqyYM={5i?_wT&kxf(0TGd&7Laj&q;Z>@6m_t3j z=9DX!{tLdZ&h*xWWq;}&y~=NZRs&V@*)?-@lW#)ID|IA#i$8eml}83PmyDYEm>sDL z3VpyOH8)fFLR7T^XR4!V!NY?nd($@?`ofvu(e-a{PM@G>xP!YXA=FA}c!!Wx0K zuNmPo1mOWtXGUucCE*c;ZvGtiNmS=dtqTXL;R4ia4v%`ja8j#aBrizj zFiM}-KRE%kj3JD6SQQA#$jC0$M2Lxr&CH3#3ZoPw3p~vz`s?}s^q1II+f0~>w0yG? z>d|NKKlU8T}I%s8NWPB*#Vi%RAy0rv5r&zj7 zk;|b8M4w^YzWs!i^*$Ndq~0-1B5#dmpoF|rQ|p3`2CRHG<9~`>HbYu6SEuWWKUIg+%=ao-A%0dKBfjsT? zL?zmxYRM`pL=cz*Tl)|%o@5-VH^k*&g|62fQaaFjJ%N;b?h@)3si;Vjj-aSDJhm*)?zWY(XHH%h3P21b`v51Wm zUb_y>XS3UI=r0WN_Pzl*X)F#LMCoewrzFBGHz&C9en_4y4w@rfXNVBz(o>?TLwiO9 zLirONX{Mjh16;cdYjHCBg^rH1)eTNryj4rKe>1|E3;4IZ{P#oZ7M$-;u^xRnie$Qq z&1tcAh}bV*8X;zh+Y`wTM!a0L%5bRi|0g}D=kFV{>byRQ~1ZAMc6=>?P zE%zs50+|)smtN(n&xeKQsBW(6Eb1+M#eg3)V`Nv{SQOBh92(EKuel{9l@LIs+WRn) zzVEfv*+GMP!E{zS@#gidq!Jl0^1KW$g$~#tIDcG&>Zsox*YFVyVr*2qX^sLT>PiJ4 z^$B+cUIb0=<7dw>ybjk#AfJZ}tS|wPT^AM^+3?{;YmQs;=6JpC`fve!K`q0>A%Ffn zoASKE73O*{S>udA{O|j`;N`$nZS`oCJtmiBi?+G>8~?kBvAl;=lSYkKYAOX> za!is2~Ja?P8JgOL^=b8PrZZOHBzRGNorBO9>CMm%hL3EG#R7^qik;kjMimLm@I* z)!V=SZ_%N^rQjHD6R6MadAWXNG|m~hvB3appR1UF1B=tnGnAr1z4Rp6ZSNXfNURRF5s_6!S7Lnpn;7aBUG1V(=_2HG#_!%Wg~}BElHQv(E@X zFvWbeS1ErcDRgxUAkBa-0+<(2#+1wG!t0T|tzu-1 zpu?c>+=o`3nR8Cb9&>SJpYJ0Z|(NcAJ;_j zb=C`oR{h

{)Lb*Rs?N$qY0v6HmT1t1bIZreR!6@cXy4nJM4(u(CJ~%&Ok3|97oq z>)5qF&6tG$gzxRik&6BS&sJSF^( z#+mCkUNXh7faY{Y+67t@o6$Qmu-&|jVX8W#jgkOI8*nFDdwY|iE9QP#RTzP$8=eDHc179NeqXm8pnII1fjt>MPK?9phzDj8P{W-$gfwQ*01SH&}l|FA&C?5cV zQsn5aO={GE57F&(F$V7c<82)5`89KHdja-bB3$XEOBnwP=o$0#dgGDNF z(VU_1*jQNF&mq?@_{>(*{qPZ&g&I(RWXUd8*T#Bg1nfs@yHi7tSh+Ri1BnJ zJ9x-o2DqKAW~$Y?-~nf#)Oa9B{bi~|H}mwS$W1R#O$!kg+2}{Pc5bJGNvM-7xL3E% zT%cWI#J^m`nXK0Sb|uM|l$SVLdAT$_a(-UhYLxHsD~!ujAUIdKNXI;VV>A79{9_~e; ze}3*stlsP4!?d!qd&0x>`5_xM?A2!Pj(W~&j>eowo%L{lR5sTu6bj0}%9Oh!IUMg{ zZ$&BO6k-ZfWt~k;5wlHVe+s54&QRAU;YE2{$9&faN=6Zp#;)WmkM!b@ki|wDc}%%% z{|42|P1WbcIF!?3Y1*D~aLR~U4Qd)q8jV%?{MPA3d%TbCgEHD$LlLbY# zr!Em-OX-;5?dyBbZan7??W{~&XwZrcNuto=_g0vzpfCf}r@7^kNa*lq^F*pSY$M6w z)lfZs8Vj`n*vr#8M&jc6R+f3AcaJTlB_!?}8)tzCbqxp4e(IYP2psjuDr_1W8Z-0l zM8f(E5E$V!&jdQ?L9N$vZ!#_d#eC;qnvG?}(HbFZw4;X6%L$6IWBCjPb%) zEK{C>&Z6fX%Mer#Pp`uwmp+V~a?P#Zi>k_Ilb%sK3Zw_O_ggfK-xL8U@Uu^4+{!Q9 zn<)mo_P1#$(-Q-@MUy1aTo95m_$~ZR% zXd>Eyr||I0mz>j`rsK9Qxmj2(l$6lj*P_0HZ*MtN$V0cu?{z@H;K|t;izgEOY>*k@ zjVACxCC^~vZwx5@L4+(4^?y(yGO!=uKCJv|K3<0!NnBI0EeSkqTW57Pl^-GeaLUtD zkWL=$?%fbr#uby7GF}D>E93`Jf@)S!FNhxoDUa!pS-k~dT1*0DJWAZIux6fJU2QHk zE`g1y{frf5DOAC0H%!jcnyOKT*E_$q29dJra8$~ZAD4$l<>qiIdSxiX=(g18g`u^` zwNqwMXBiRugZMgFkqgj?l3u}_*0I6h!NCNV$}N8y!9@q@>xulo3MDqW>LSVupT!K6 zVM{gKM>PS{q$;Tf# z+M34p#H_{2Vb4HJW(aTx@>EA^&d;^wm2%O5JKg|~gaBjt-umizRENd6Kr~tC#5aJS zC#{Wt?^!FHpW#3{o}hV&++TQG&*xv3{;*ew=*63>+S%%d4cr~4cG}Vf)hb;hcwOwfKL8yZQX}>p>9Jt4ni%Wko z5h+)`B&iPTD#C1ky1;6{|0p0ckWqZqIrnn+`l$0uG<#lKs^EVDuYSml5TKVqt5ff^ z**&-|ix}}snP3n?0CbUIQ1K7y|M3F!k~(Znw1Z&r4+e%7loa5CL1?-CwIJGrVa*i7 zsgKLB!s>atCla0*TA{>Wcc#KxK9OkG&Nm-!!|lKl2a2lRBL3{GUiU9+^}|&RWKCF* z6xwVaze&wt?TggKt2%CQ*J}i?6T<{8Q5@J(Y5AqrlM`A#=U_sL>dD$Iqn8-W{}F&Bo;OevAQT>egA~sFqmWLlSfG&(GBB$4 z!GE@r+LEslJfw(lI#r-1GZ(z$1+D=}n&S#n9NQhWkAjUwiDf*{)>*-=v}?HfXR|Z( z7OuU3_r))rBw`#NuvLQn&+PQ@HkhK(>~|^H|3rP)xr51>NXepJZZ!uDPu7JcT0=)q zySf*|<{c4$HfddPRQ?Mx0eCjb%}~U6STYaBV(htH5KDp!ZJdh9PT~*KImE??_{f%Srm* zDgNS|I0gejEDVw|Do+WbVK=7h$v~?>yF>IFGL1oKt~0G|ikJK;(yF4g;-JObxbAqCQ}GXRsFCYD08Z7uUj{=Y(BTTGuNxDU z5wPQb?=vEVoJ|03rl5jl&(GolS{@*xjY<=qsyppYZY#ETF+Ta*`z1C$?u?aGdYCe2 ztG4g#`$+YCZfTWaq z$b?Vu{~ET}zKQ2u2YH$cqOBpadw0rpyAR^c85m6J%d_8cz(K&m+)nk<4#S3mF$X@5 z>R>a`-tB;*GS1csDQOuOL0El{adM2h}n+e5Ud#S8mM#M(8iyLVfWZ>wjp7TYR}*- z^mjk_;643ck4xo(SZ{|;&b7lc6dxGcXF+AZ4=AFEO50U{8J8^{Z%uX<>vy8O9UdMY zQptjeVx(Cf#DJ_pse&!S;Lk;cD?=1+8w!001)FKN{X0?oXne^oGV=s``VT){s z;O1wUSD4yNP0~(PLN6t zN9?RU4uCc1$<@JshoTAo{c%PNusl}O{rGQvfeEcuU0odz+S1q71D}GGPQ% z4j@DK-=PUJK2G$U3yD*9jr+e=c}X_7Vk+k8GlOqjS>PM zW8i&8;6VS*f4$|zBtkX|7@One1$!KzLPm9v@!I_u+xyQS4fdS_Dan20ZtM-?+eK0Mdst2_zX(m zqR9Lr6#VZKh#vkXX^;sOLUO}shM;D!&xis6`(OL@2_h^u)~ImA3JP^f#d`msJS!-s z7f|z|tRuI8`1h+#>0`^voB8*xgNXdL>)L<5aHlr#^`gGDo3rE;*MyN+Ap$>JnlAh(d8afCimbl%uydoUh&qVY^1tbplWVn1?~NJXU6k zrVmvx6ozefb{0fT7Er4p@*b;$l9CeT6an%H1~&FGFk3SrpPXE%;eZ>9lx+C2_py2aK;OET>sg_)L@ zSH)%4mmm#zbrSHD;gsSs;^LoCm7!H%0*x?eJW7YMr6fFC7_46cy6y}S5Xr@p1sMqa z5S^X|7Yt2LHF#6N35f(>Gktd)n;1AA62Qj9xC(oKjG9_PT^GEkbTFYv!ulHp@&hGI z*Ea#5K84xX*c|9&8STM-GFI}rAZ{Ifzttc|EYNf+J3yq}e z;YVAO^ktUQ8Y+;&?Cman4P3&Iy1$rsque?GQ5ixY!(&>o4#NgC=GFdF3EDyjIEkStk??|1hPW7ujQol6fH}hN6&p&x6@4fycJx zb(zbv#f1gL%*+fzSWfOib93{H7cW*iot&HiS@Fa29B#d^6MXXb_lGn|Qh}#*bgYCD zxY>}7xD6__2b zJekK9&DGV_domoRU}1SV!EvT5YSy4Axg!Od{wRb@Qz}DmbEv0bvL1DuUa@5^`3IcG z+G02l!Ze&N^G$3sOGkcvPA<9+P#5 zl4Z4+;68Avtw$Bip3>6J!uK$_r=mw+p^*kr-323(;Noc5g^pnfNl9-B7n6!YrIx6rP#!%4gC45=c>4;;jOLxv-5YVVO%m`O z(Cyt&f3!su7Z-DbnE^ap)KKK!jWOLS$z&XfKC+wy=;rU~E2LwR%Ni}z9#BlR&S*vo z2snc~^I50(xnpvV(Ylr=X*-S;^ZiGUf?uT?@0sTVB~QH9eM=@*1EUc=B{lU~ zXlqN$oI`cdsM)ne;&OLefg;ZMJLTh(8|5#pr=U^fDKqnOM!dxndip@PctiEzS585l zgn~kJsV6~TV4yCnDA!QadSQDPJOww54okp<7Y%QkdL;kHkFcZqPeRDuQ0OrU11DV) zkkDYEZ^HK2xuM~<`%-wUp3h!EmFw8qZi{E~(Bn1YZ{W(Iiyt;w4~&%SA)c_tf&h<} zrBLhr{V6{*$D-N*p$eQXe(dJpz{#7)^lej8LW0|7;Zyylt&I)*0s|IUBrBLvMGHnE z5)&r{nA7~f0w4j3)$_0k9bd)3JHEp;F>|FnR=*xZDljH=wUGco{4@VT8Ch9y@L?Kx z#&kAA{r7S+IX*tHX;ur?HzX^{Ei-G?wmk5XJF$nlR+K2XWqO@*bK0FZCl7u!7;037 ziL!%ISMos!&ts^}zb!Zn`>r74p@fS`^sU)%7A>W_H&}38)G;(C_|UKP-miCn)$hU+lK_ ztsg4oDL8D7HN(?_5={t;9sk8n6O!NLyFP#n=WCCrbY~hqKzco=O)X@i3pIjn}W;K-&Rpk_`lHfJLPZ%KM#Q0apSE z5*og?k_g5aO@m*SyKI(?L;_-8=Aocl1 zqd-ojgEWay*s^e}65;lz4;;o0*fFDE+o6Y(mXk0MqR%0Cpy6pisi)#O5N@O3!cGqe zyAC*IM8Y%h&D8WR%O*K5J=BLJDG3LMpRArpx5Ymx&#&pz(bGqvlexWvv}1Wl*!lO@ zFqI}xSzRdon^O39f8%!GDXOJ_uq7JDqU#4X(zoarTN3AvNielDMtjczj{5q2WRK}O ztdMl5$r%M3A?gQpfV?zrPOJ6c<$Jk+{zT(UUmQ^k@)w2eB*NrAIHI2fUtRzI{)>uF z&<_;d)zxKib>69o?B6^DZ=@rfGSa?(|31FW=(F;uv@zccNA%$4tJE*3iPydhUh~I; zuQ14&f^T|>V)UVzlzjyXzn0>2402vytZSI$FKA!{=?!JeU+ji@qB`AS3KzrCAv-7M zMhD)52Zcs`Pg)>o^vCQ^eTivQV zw}2x|#nM+(r>o23>I073ZRPcCQaMLT~&Jfg}+=7BP z3#tR_%FW)MqkPhV7!E~?j`in{P|e&-F@bzr72a@dI_<1I=$GU_`12f#OfUfsn8kLh zGH`U9*6W#O_B@r$kc#_d+bhrt-?lOwx}gTJdFBVN!-@4KgAb1RJ5|+A*n;CpG{Fvo zOD%;EE*JqJwl}5E=pqzi9JD?5B9c*p)|C6PI%wKTR zV2^>bYUN_IsuN7H%j3$m?3}L?uBd#OWFayy(AIYW1ytg3=>khW-{m1DxeLL+FW^ql zD+-8QP>Gm_iF$D5wiu;&<}Ku9nqT>U=z0sND!1rgbOQ>C0!lXm($d|6(%s!icQ>e% zq=*R8CAI19l#-I%bR*rJ`>lQc_l`T>9q*lS&Nz;9_Wt&_)|zYPZ~kWJ(!)g^xOrh6 zpV6v7r~w93tb*}k^0YYM7-@iKk75jN>%`ucK+xip=qCV6Tzd9qMpMNaohPD=I3c?RX$XMMV&xyV|VM zsHmur$#>lZ^jTmM8FaxfIM9@VbjX0F+;+2+q0qVg3p7{!$Pi%VXh#5{SeI!9Kka~TVgQi0fhn90qP-W}_6Fd5W zyb%d`u|aqj+;9LbI0sM^Sisd51%XU6kctTytO`)QWSVGSw%#7B_0whi-aK`l)X-Wv zy`Bb?6d(rfA)iiHtd!*eKNqK46djDi30Wve5Uum%|ra5??w548iRsb z7~E6vE&nLQh5i5iL*f<)=qv#TGEi2ZQXCo)1SpT2WdHj{$v4OmHh^15AqWY54_gw! zHe~<~f^x7wK&(F*aQ~;1@MVaTN84zI8+Yxw~!ZHxd~vRna1&+P6d zvoK+U2MBN@B8dG&yd#@ZssoRt=g<bXyf-7q-`i5y5X zq?F}odJeHhNakQ(wQU)WJ#@zMI z#>JT!c-T8T|6iZZ0-xT?+}Z;C7#JTud_d|Q7#ILfsQsxsBcn}YF&Y~O$MdEZ%rY2k z8%9tPe}P4AeY^ba0*l2OzHFm(DeyDN z?tUV_wxz-rl_b+McY6=cu_u=@{_c`lNMemW5(**cj4H|Ms&Xj ztP)*0KQGe4f4Lq?9mfnJvNm#srxQ4N+?lYgLU-7TMst!oEiP_zM67lWUbODGL|K95 zDT4^QZ3Ou68L=>VMi^lZUgC&E4K!3&iA=W!H4I9e(-W?Byn4o0c+SUA#?-ri8iXE2 zjIuK8^qg229la)90K0Q7{8?mo0k`sgQOCY(j*6``J5B$sGtyHCYll0`Kn!xJ; zBcN=+c%9E0P|dh+->~cHN$Q!aj(C`UAf`B2xk z^Qs!|7}dAYA{6+W0E7Bytk^QC5>=eS*vvtlm@W!3KMJ9_K?0gGR0D_1J{uT=^`r2f1|-{hZ8c-edz(@Pq4WOO&Ab0pRNj76bo#-*W1 zB2^T#rsnrLX|=(l{W-WQCynSf!tN8^sjowOaje7JH{9bk`z$@TMgF8EAaxi>6>8Yz zy;ge7MEjXiSfTtS?}wmV2z zgL(pe#=l12uIad`yrF`rMYh({0dvu4vt|yssBdOTU%1BZwWqu9mHdrqCOOMhv|}F? zMG_;Hb9iK{h8u!m*(xr!5;4*LeNy9SlZ;hW$LKLvv!zcoW+KdArVS<;jRbB|_LY;* zZsBF4fL}>1p~cWL9lvK?gP*U+?dLQ!W#sl(K(Tr@PF#p8J0}IE&AtV9nz$zPs$9F+ zWOUj9vpAH#+J>3?KmbVZ7?tX@k@MEK&&jSpIC9pcOrdqIb#@Nm@JW>$4+RPYfW51yUL!f{~j4N{qk z_1_}y*D2rl_W3?*9oNDikN*{PDh-nkDN;i&E5#?y z*g5Hi?%r?IGUI)o+#URTbc@^P z&n}~v#U{YT4fBWws_XNJbW7!md;*spX!v;%-t%;yK_5=ygk%@XjTy=n-3S|5r+Vm& zW^+}SyF-i&$>J(O4GEf&T+(o#$}Kx)0y#C$n^5pO+y}Zq*h>syHc=KA?Y4x8Ns-y~ zd(5KJxr>9LoJJ}K#ml0o26FW*QDz!+GCXuQQjc*ZRtE0MX0*$H-tCA(fI3^1o;;3h z{h@pt!wiC^fsbNl zZ=qAlx>|}FEg1Rv4|}eiI7Mug(}bqaO&WAfx(GUC(r5B`2zMh1U-`Sn~hct z{L;s3P&q(oC(Pc*@h$S(QA}(|C`;MMz{(%9T*=osS-f?-4J9>g+4QO98J*d&xz(kq zLlSc`>~u)W$KHQ!&x(aB55DbYfCCuV?Xii2_w5fiR=*=X#v0pA0|b7wWW(kt(qy|)%D zK35P~tMTi)e^QHS_GrOx(G8QaG?o4HXe7bL;%$iCD%uz>L^*F3LYu* zB-R%8_tbe##!#@lXX$qKrA6A)65Y9@0LB*ty7Gdlp+{=(cjwmY5qARrdym z1iuTfFkYtFm~ejo)T#c2P^4Qfi6_cNi`%k@#AlN9z-EFGLpLFm{&Ps2bC;kHM(6IS zs;-t~dQOn=hg*gcru@ZSsy{Ej4D8wvmTLKFpT(*3F%_Khjas`+?~?K9$7rJ1)yOXV z;}M|)q6~K2`}W9pC>>TGpqP0cOe!-nC;xB6xsu+MV9nC*%qZ z@)j5A?kyeje_nk?{m1wzz0sz#sYU(bZ&N&(X9yq9s+yE(_L`x4-oLktfIv!y9VGE zum|&r@#>EFC3!x(8V~MmRD&)CD|LVFV+J( zxlFX5<7Yp|ce6Y!Q7shke@LI(wa`0OEZW%*T7|DoU5P|wAKCY_kRA{NBmJHvM`gU+ z_N}!6zex8cR;|7v0scGl&B>9?mABWFwYdIr0Sc7BV_fYt{lf9ybSkBMa&AVynyhO0 zJy2R_1!CrsShU@#ZlfkQ&$u7<3X5x8XPtXQE7~esBgQW{AJTu+A87--78OG=S-K=) z^#rwzVSd=!mmD}Ez=#3O3%rvXsS9dx$DI@JWP>3fI>0bi6yZ$x8sJXhx(<3^r|Aw{ zhR#d-OuPlR271{>cJwn_!}g2t9PO8tw=ybubzzo8i1SbPRjdLELN;n6T z#j+X5OEibzb8yt9)a%-=GP8^JpZbF3l)seI@-d@jBMozM3i7ge=|*v5fhZ4s0FENi zv4M$-TOxELaU(nTj&%a^gdfU}GIUM}v4!g=Z6|#VQp?k_b8DaH!@hKBL>rcLg>}A? z3k>ajZhuK}+0r{{q7NMOWf786)BHp)9liQyKDI>+x4WxItyUeUg}S!Lx5w>!sd zWtjiddh(gHE{%$bHhwQD3tULL?o{0B#l+T|CJM&f`_FQ-qV*uix2Hui9@uGNIkO?- zNlbf(C*^!SAN7S|!$?ePrFebd&QTdRB>rMNwR-dMqX2yW*O;E}O`Co4N=>O(N7~?f zVDD(Q_e{~sMk2M2PufG5YH$3~bT`Yw(DB+*uAZ%StwE+$&zMutdE&`9qoB7&qoqC} z9QpaJ3Jm;>?;Nid4*UPt%lN25=)EBl(l^UjJ`dg}OA568ZJuW+2Yty>5(bYc(ywx7 z4EgaG4~VW&B!641oHTl`?`}*RV0^ro4kpd}rD$PvEykFp0c<{|Iw>tT)z9BYUM{PZ zdV@dnRBZoduE5qPmUlZ#&LX9MO)dQ8wBnXL{_{`=LH_ht2POX&q?nGs-F%y) zn+TRY0y&;+`&=}(zFf5O!p>yV^yxeGuym-}ZoU8X5E66%xS7*xwHd`p@?%tuEHa+y z)ESAFEnk$5<4ni~k8<5yvZYpbwJoqm#e}at+jzhJPptfn{4M3|O7+_FloQ@6s2j!k zqK|&G&)?yp_<`!YDj&EY()Jr{g{S(w#;C+KKLTDd04Ej5l*w$n!K-gWUtXu+G~Kxa z-FI~K+X(WJ0$F7Joi@m!TE#(>-6Da{g6Ez9Y&_Z8$^-DTJ@lp0Df0`H1R$N_L-rIRkze4ZsiNb!7RhM@98NZ^ zJVld#LEEIILOmIt&0DN>ZFet3Z|`8he5>ec{lj{&IL{bf|Nc}vIxFqwJxJngVE}R5 zUp@DpF{sU8J#C{M#U1(myP{b1oJdgi^3nI(+vC8>l|fp&(!5Ch<_O+bUyfc4KuRgx z6kR1<4{wvpfxR-mN9|dEa_sm{H6lNdpaq_CICgG3pFC8=BL9hsARAtw zc1T=SLj0&C_zpfpgm_35dfiCKB;n958tm$G+Ydr+REOcDu9&d}(juWy@ZaYVM7GEEMCYLnOh(Xpl0WI92aF z3cR_v4l?_a_o%Z*vbe{}fsiY5nvDsv(&*IbzS2>Uhe)Eae0&ttIkJ0tGW*dJIVM`6 zZv{=~)I2>Mk#UOz88;A8HTz%rg9zY)Vjza-5A#n~8rBTN|9vI{nHB@(b~;f}(dFq+ zkDno`HZOCD-Q`sj*hdchz7+YSz>_pfHo|7=O#W_t*5*WH(%kS684ys5slWc0av$5( zgp$hXv{gy=J;>3l|g2ay$|0X z;;hAr;t;Rj`1w&#;xdB617wlu*xBO%INup&;F$%xj7sMO&lOdAAPE8?OaSM@CV=Va z>>*y_3<)UTzhs{CVw#%x0E2=^Cg-R7L~vRTm%D9`c^H6J%`8pN08^o8NwOpOGctN% zX&HlehfD6COFJ`R?mlEUm60`VTLNPj|0UT#5^XYm*60^~1oob6^*_*nWei{jxOjNZ zhb@=i0PO}Sxorbf-qWW~(|pPhg$a<4uhj+IDcm3>ncaU3$~8$6k@&}8(9R38g^Pr$ zi0muRKU<1W&(e+0@8DGOw(EhLA`F7A|0Ox4`z|1w>gq>p6!0X#7W9&UKnhI1KK(aD z4ba;lTZ|2+R8z+ajTK&|iUx{+-7tWD00qDlVK6%Tzaas0c;NdJZaP@T=H`SRzGQ}F zW&PJJAsD_ln3ZR}4z3?z23A9*1MJ_QK~ML}ZK>F1Tt z?OY7q(*MmGxncdEMB>iVeG2d!0LplEe;BCb<0D{L#=EQ!()R#cAXY?#;1@4=F{qjz@R;iT9Txe| z$RQ@PY^c(I2!IqV>@63{rt373xF*G*Igoc7EzZdW=wOru_=VG)O2xO^T*!iTxd}!5_zs&-hr6GJh68n?q!iczpw?n5cTT4up0%2! zyk&ZYxCpAaA#KfV1o;Dz;~N{ifeWNg0oS`rCu!E7>jTDumetE+pUkv9AkzDu^Ui3X zvCtA!qJgY~3QXYfY9wo=Nf+v(*IKl(R)ti7Gpy3)5=vSugzY2*T(O327+RqwVvCA! zLNJSV=SzO*G_CNhh0?sBz?)L2+3^+UQ9-0&p;~2<00j)HrNCDP(4Nfd>hFk5K7atA zHAPU4?7N@tvi{(C`hco)<9gU9`0Em=)%S_~27AjHDN1WD85zA9TB<`;hX)Shr-4;F zB;rD92C?{KFA)u!h|$`L`8X*xNrT+l(Nd3f2>b0=gDoC zh}WW(v*(Ng;*L*`0{4~IN;eYHZ0};rD7?5YH|A+NfHn(f(K6V~tIx&{a=ai=+jVn= zbCTR|Zt9DC_2wM=DZt3)+VlC;1)2O%RA=-&VLt~v@<^-gW>R7RlnshWY{ocS+?R!I z^fBO90m)TCSyPl^r9eN0w^D%teapvi|A05k$LH}kIV!woVhY6ReX!ez`qRd&=M=^p zN}@jpQJ8p#e#ihe58^pQ-s4$DDDM`G z1c{tk#zI%Z7npTx+Y#Umob&pmo1a3G|NON`<29E{G}$oJ6v!4T>0Cg;_x&*0FEMOK zy}ophC#ve6&On^Nbsr)Z^}AwgyW3#B2I93a#J!_~A8i=)PDA1o&(n;{RZyoA4NoENd#6=wc%%7CuaTscH> zhlHy&^;pPZaP8cr%>8&mPmrOCU$5j{fVD!(&aB#b-_|QCs!zbOp*0bMn-N^!zMXyA zReO&^b#*-dGMJ=VUxece`_E(hMoG8zYw`|RDgVfg2qqhT1J#A~tHSKOcfBGKCNs86 zED{VQPZ$`L~bvhXV?k8i|X!+ zSW57vhQDLU3P=mkc>=hjH|-lvhg;R(V~eUY_2 z|DwF#>kHCykQGG8FeJy)uK9j@6n`KuUl;P%X5zU_#Z@dX!JY6)#@f~W0!}ywNpkA0 zoH(F>;pMHH)EA8g^YZ{J*kU*Le@EaukxGWl6Jnqn&Q3qGdN?%aT^;qQ^51yn-A?VO z75CN%wBv@AJj^fc=-MNXuA(S6w`y`3bzz6XD6U36z5IqJHT})5Bx* zhn9Q!_69)JJ4bECx1-kOa%jlHO;R69CV7ud)h8dhs%(e{Q`%9g zR~aCy3N>``_lf_C4gd_`NL?rS79PmkueQqNS{ILyrhSex5ma6S=%yKiNM<3&$7 ztqKL3$n@KSp9CJXMtuKhmXp zWzVF`f^E#Mn{&zh4L|kg{b1JmTPSEV_1O(}@rwL_%)xith!+P)^DVx-0AVUo21CKP zh1hZtJJheB0@VQc(&)T)OX?c;9srUEc(!rz@m*l_MRMw;PYb|e2m>hQX%C5`vW73RDijT*FOCJuzm z*h!XXnpJ4iGw{QQsDy>n0940g`Qn~P z$o)71XhCD1%Y9}Ggf~q_UKU$f1q~E5J1{bfqZQU#&vV=1mUw~|go66~m8fV2U>ie# zo59rBjt_|jh*nzAwoNM{Qja6(3Hpct5csZxq8(r*of`BJ#61AfT)Qn`Q-fa9c-l5< zHT;i%Ou98E6U5H{tK3{_ToYJWf+=Br#m~S)T2kxS3K!nK6a^!u02Xg?Y2dOd<36hC zpfM6)p>k9ULGA(|qaOo;_75L^_w=BWR%yT4aufLT=g+Jk8FX^um%(gYTw?2UBe&H5 zQ*w9rybY{@-_b9GM?K0GY#=Dl-+tN#2Lgte1N)ExfN@vLpRo{hJkA1XfG#X9Dk{pu z=yb+1;{%ug3H+!k3k=YMU^RZOi#|V#^H<~l^0>##Sas?4DTMx44AEf&FS$iWl8)xg z(qd58vl5m^fnuJOKof90VXe{6c6xU z1AZXnm>D}BU{cuG?KAFn5RSKO%0OKrY`1GI4r|3Y?tk`hM`r))qtS4;n>XnD z8=&&KH#wZ6?%okPlAo!{E&9jZ0ZBTF<}6G^;xlk{2v`jV5s**>Y82pn5oBfv0{K7R z8LYVM2ap3806e37#ts)3clq200TzWIhPIh=umb=wz!K>+d$A)po-C>mPOC%fwS^86 z05(znQQ2MX*yLF`W=x9G4aKTfU9*$sx*N8Xy1pLb7b`V^`K-lv?TKg(qZx@Mi46oU z_!6!3M1~>bdCwm4IgAJ56(G~Y;2%Z6DvOv} zLE(S)5`nu>QBgsR4*}RaPKWt=mL@poh)@?mnrsFNSq_sz5KUWP^yptJuI8)NNW@^o zd7s@9!e~h_0b>Z<>WL5Tz&MPu)1mL%&8kWN9Qx8b$F`r>v;9=s&`)S7CY%0&yi`pB zld98Qo7z`Nx$0KC0lwMjgVp=YzU1E?)ZIt>xTkJCK^?`o%vnWS^POx?`dDmT>IFQN zlw{u9K6@Y7Wg#%I^$T#X5U3Z$RK5;G7tWkx+YtPI30@Hhe_&}k1S(Od)z2IH;ahyX z_i;BtQ!ys6I0C&)aG}2K)g8@&i7DUm$@=o7v*yzo6_LenVf^yFC%&{VE&iVUdHtK) zJ>4QUQGIfy__7>s{!8>qFu(ebjBM%*#hXK66{Pm%@6xDrQXbxE5}Qx1AxEhvCoTXC z{<|ldgn^m4+qB}*e|Q}qh;Q_6cE=L`Zr%Dz>#@`0A3_jd{qNS>+Fto)CS)v)RfUE5 zM|{%#)tR4ap z6eG2z9Vx#VN@mZ@3vFmn+-E_@Ti(%ggoA`YZv%ud&{zy=dF4QI0pX*Vc2Y4LLI`+( zUpbeh3h8`KTuL0@Gr2a#7PejQqo3S1KWV1seK>$qVhGKxjQJ=d%nB)nUMe46n3g$n z^J{63av5dhOf9Tyf8tJEjhMwh1XP2cf<@Br8uEc-U!Ip23XVJpbPN1(Dg(>J>}M*? zYb>SAyk2qdB8;E8L#K1)P=T~chw@Z{XaYI_ij21EDUY(kfpKr##EogYG z2pkPB({mnKXexcQ58P$UYz8J{FfO=D0E+!TPRQ2EL&Y`jU$*!B)q!r28xxFpoISbODp2jF6~a10 z?%+fbz8A{>pe+*er0LVE+=$?`Z6Adgxy*O-ktp3>zc^aRhJx>b3nwx1+n^BmIe%M= z*_+Sbs8w)7&~L9zR;>B^bMv{}Itkr~@dHH@ySz2z$>V}Gb1cTw#bp_HlQ=jEeGR{E zab>mXRUlLw(gshy=#X67?{J=>r6wxg5X-}iKObgfqK3nt@zuf2Kg9F9MW+qWJt;po zmWkYG94|E+I#dwl;Y`0OI8`f}a8{kn>-4*!8Hb_JB{K6C)f)1jkgg+!0 zA3V&%$1-6~_qj;=A}4ouJigJ9AmY0+*E4KS6$q*fymYzr9T-zxaAQp|V-r7dEWumdp z|E{X>2O<|KU;9F!MMYrn4hCpp%qu8(5)>T#2=oz+MDs3!q6YSVjg4|;o$$lw{gQWSFWPbi+2CM67|9wcNehEw|yNKIg|f$9fvJ4Ev%0#XjJ*;ltq@B>zOUE4i#cnzNTla~?A zA{nCh{l8Fyh-;E3uKsOy;b~3O65k;eU*ApBEVNMz8D_Mhl^glJhkjVgM&Z?#j^NH3 z2Y5XPDz0GAHV=TZ*TAj+1E8+;0F4Z0LFu<&0}$l#5>W7OfJ?W&Zr*zRk0jmqfZw6{ zxaZX*qL@(NwK<-%Ct^7KX-215w{E1(^ink}2x(01&BJ|gFIMDbexfzh_4ddk5!~TX@!QLf;cS0pzRzYWxg%LB{7hlU7^`ZT^Un;sB-_0<23Ka4|^$ zLKGM|+~8y&Fi(D%UGo50T28L8&h1ILUWUdu-*#VLd&agjb7$WDx_}_-bGA}g$`(E4 z-2sule9XoiipOA*{|qF1$OpL8sVt?KNlZ_a)eYrb>;|AV@EYb=wQfwHD?<(QFGdD- z3)UWEK)u~{-Jws*k;Yb0!oqDqX5c$+i;Q##V$9G+?VN)x&y}E5&L{%my5=T;n7;_W z&VbMOP-DO1dG z=Ou-YO@4zdey_A*6lrXO5_E8Gj@M&Z*Agw&ib0%}md0$@RJz!e0vGAb6P6IVIMEEa z6xHL?ShuS1(^J>rwHw8^=h1iLE&~Oa?F{(~(R6l!^KS>tKA|AQvbC`RZ7EGw4WNt2 zO@J)Xz(a?3LKbmsR-}ac{DzA*ndTfe*QePG!F48RUoVKi6877}%rnP}8t#*$XrIQH z#7TZ5KEL7~mhs9$mlBSKC3^XzXY5-gw>JpeO=4VDCC$}RIDemMFLNa6x`T3>>(z-+ zfdO5)xvxPdobRX$z@5|Z>+odb=Oq7cEI>Lw$a(d6)LQPQF11ybbXP7lm^vjMn6vx4 zF^<6fh_!0}n&E%oc~o3(&!AmWSB~60{^< z1(J36bzvP!*hkStmt;Ck$F)NerK{M50FOhE*uVW2`Vl%@DqowVI}E40sr{<^$OSZu z9F+y$M}j^h7tqfH_`9A5Dj9)>DMTwO=!6sjC|l6z!`6qmjiU=iEL%uIn^Q#ZfaxXP z@I~{ z3^PQPV<)@Me64 z*6bMLfgruKeM&7nIhHnQqav%nyu>J>!k6s0Ruaf7I)>*k5tnYW;$(;iQX!kOPw&Gc z(QRw((CrNQ7*0LQjW_YMWE2*BYH2dy{|FsUhugf}KPw=^U5 z2Bh<=fUW`)-h&kL&E{(JT;&1jN(!kjz)LrrPw6!^2Y~$`NPCtJ$)92h1J&MyWPh7?=#7$m9dL6UvDpp*b7OAp~#u#&9w_i z#)dhZM5#>!LrI38Hlb>$3aO7;sshn^q2{Vh-UJtxxva(N*0`=TId+Hz24zl zKQ*|NsLxhe{!VQ9Z?kS)S!187D7sMbQ}-U5cU0QlU0e{=rPKGPO0A zTdxfYG$MBT(5>)V)_;7i9?$Bpw%Bw=nRH(xEz=+xBCdxV1K!!#FfQF6cp{o4l~-*1?_S zzrRdV95<(!Q~pkf6QldfhP`B<%zuP)y`{AL(yV|%Q35oRFOH9By3A@ zra91HnSORnjbw~=zh^0G26{I@;L-YUV+;5O#Q;PZ(cw1d{$GgfT|`Cw4&GJ4`#+%X z(#!4I@w@FIvWMOeJajK#Vzd~fEj*00Hib3bgtpjE+Th)*W{|lr!I%*;9j21qGaZm3rCCbcsGG{0qmv!0~ zS69Y-e=nC`+vd{o*IL!iE7sT1vDK8#+L>1JSMN8{agZCQ6|b8>~$Eq zrp~wH+(h!JG18yNp&6NvlCikAM$K^)M|U*0Nd9Jer`zI1r9IM;jtjrVJ%&KU7rzl~*X zv!ps6Ey<^y_CX}OD*OaN2A~%s9?e5XION6tn$>pzztdb1zLltL$UIUt@EW2RB%a;A z+_IUHriLjI|83n&HMm;2xboWBPfKMe{71J$B;zAHz>g_cv~wp)r%UrV%1WmX^*>%z83bX#ZQD%|+&c$-oS=_v?Osoe*dMdXNJ4ii<^$_i$pt1@|#l9#>2*nKg zWJcg#psNQ0s7{jsKU3zU4Wi=~LCXMoYJdFb0ctBrG%%oUs)~CvR7ddJyCTj#VZ-6^ z!#$e|eG~6pS5**|exD=RNyEiaH1TG>{=n^5Q}>m=>f`5psC0FxvyKOh?lFNl*NCRC z2qY>Notl$5;+?k!$TjqJ9!BjPxnrdkttSMMBOBjO3H>ul`*^#joGd$e^z*NletjK# zD%KJ4>5U)w%?=P;Uj+Wsn~UR-JW~KV_20%ug?axs2hf{^7%T%42zo-I?mOy$MsYTp z+eQ?4O=Cp@q%}i_DKiPFELf~}_8IWw<#p)*_gg~4t2vQcvNs4TCX%sgbXR$+O_Xwh z3mge_pR~|fpICSUXYb|OMj+{<kqLDV4u3*rIEKY9D{b-t)}oymelq9lDFP)?nQ6 zky{l&VH&?GsK+NHfH6TP<0iOnmFqvn7%08uV-EA3yk%WWcS zpdK8-Wb1t&-(@GP-5J5_adfLyi>NOlDMQue`()cm^K4zP9wYKGxIbXZhbJBbyV%*R zbJh7@m1_T86PTycSEFDU@lbcU|K|PE>$!p(b5Y=4d-pRT4xjk;)+1kHiA)UWj2;Uf zI{{?}Pt-RN*MV@p>DN~%<&u4gX5fYQk^I0=4{eb90$2k;Rayv|Dn}b##njw6z4IGq z37UwJFM$anhp%{@HHJDC^S>J|i_yfrGRwI$83N(m7qNU3H%U^XJw_{+*1>=4Ah;Q9 z&y@~vy-i|~$)^qux*M~3*z5`=zxtizB>6mIW$S0Cuy(mfOsS(unZV@d4a{v{LDpze(t*B?3dEhU50fo~UaIOHc9`4U zbk|ozRTnrEWo;FT^Ash6>^gaD7@$%Ta+Qxq5eH$ATIz5kFrjNyzK3#5|lcziz2~S`mPiD_Wg(j)cJU$6uAd88fzzFI9_7Y_Y5<= z=(TXS_~K09=5`@QiwnU4f78A(Ll+t@2Q%F-gUcddQ>}P*wD0cq74fuk#tRBP8i_c1r^)xqre=o3Q577oCw5`Ebf;W*DAn}4IXPU&yJ&?!_V<08Yr2(<{iaRd+ zlnz|FGkEIcVAy#x(Ypdq@9PUEh9Q5qpIvTu52eSMswV8*#RWV&H6ZO^2F`1#&E^Rp zR&nlH)oHxBBM2%%lY7EzbG?=#sEm7cHpltKfKA2cu|({}Sff9}MWUNKxjM}r_^cV* zqb39DeBTJ#{>{^`f1L)WL264jC?9Hj`1^U2TMsdT`%wH5M4x3SH?vgyAZ8=kt zA45@^!dj|cDdjdP?wf6UCklw8 z9B!p;94%T@s@*tVU_w1jX>$N@Cz;rdTP-YvK6lGi@qoM}04vL^=PGozki_`%0T##c za6DvP+9zYp@qXNpr7jUC#6z=r)a_s~%qWe@Q<{{-lXnnKG9}umAY2cmj@|qHJAp*8 z{(ZUZ`_DS>h^9o72_9-=iSxsNi-|mZ2vjI=UDX_1T`K#A3(mBwg^fQuP$s+bmHiPd z8vue{Xo{wTQf%-bct6z^dvrZ~>gu!*?UUf}Go9QCe)UYm%!8a^O+D`O!x#IJIW1R@ zC({!so)6oy1g#^NLf78yYrdQ2b-t~?w&uidjHV0K@)7=n`$+eRd<2PncgI-M9B9{ykGRcG;4U6BMM%=#{Sm#wQtdPKFT*BaDM$YtVL(t zqVM+F%S-4$_|kRcdS0nC-cw}lzQKngsdp-D0FPa7jy>$hoAN?u!os1kxL5rw{Y-U`V{ z_1{176yi{pi0RtI{$Zr_czF474_Sp{`G)_*P|MtuoMm3&_vfuQ>Ke~f^fkXkhmjas zZ8|_%YZ3!{tOOtVU*n_l5hK4)81eFET?%~?^1P{=^l{L)T>GrT0j*CX+P5AB7~6XS zz{LRMM}QWB7!X+D-FG(uWa9NE#T^*Bx9rkzbbM@m$mrqaBE+KaGx5uCCg7w(TYNMx zxO=0#&B~C++X`z>Tj+^zD&0vPMI#qYc5;yc(~P!M+$#-XC8pp1sAHY(bVmA64D1Qo#xN4Th5$3i_hEd>$l9^%o zj5WATQ!ayURi4MsAs$ca0kI!&hC~p_d$G^6jvD%wqaz&UcdqHdi*Gg6Oi0 z%j)B&_3j_(OuwDH#7~LNu9Ovu84!~se;ky1`t<^BU>1eabYy_3Xr&G=B9gXz^~crk zal$0_rvJ~TG!d@d$L0=Gjcil)LSSg0#X`|!JiNzDJ}?q>;2-YyHQ2zY>0GJ`R%ULZ zufEoZTfQC9dV>2jaXY42-z1U;nn^?Y#!a-uYe$ z<3Wl%D>E1Ql8SR>V+=!>q(A?BO9vm>md67PBFL%WcUGG=u@a`c-$FAtkkWN(Tbt)n zH{Kgvt}YaVQ-a;UG<{LXcZx_ieQSOi>EgPy zjacOo?9Cze<;(=bGb1I{qiSMu> z-J`sn4>@5attb`C3;$S)Dc>h!&Y^&z(B`>$5vo`qE&8a+VyF{vZ6HBFrFgp4-`7{O zY@D4_IV`jY8C{O?cj%8lOhx6FcCX1NQnb{U1n&n0{6$I~8X1+-!&x7`q3s9nY72cmR)Nu z?9&Y-rNI-ws_O4wry;)`qcreO`m|}CeebA|uzqxHPVmK88oU<{nx|!aWiKtkvbS!Q zwxg%(~FW;0WJNTlJ;M|*>I+uo7P2Xx%&{(r`rjxDFOwdtm?nc+RM_nF(tgy zr{`fcXyWCb-GA}VQ8hI+sHwHi(6YXy6R&T0C2d#qFD5!`VW@d|jrvw~!3ZP zFzBR~yxuG*YOWS7%@$JHgI9btDQDcmd}ux=T_W=dA3G^REjxcx=uqIT&5HAr`GTXM zA46BwFeG~n{lZJWvH-TwEsLhPaHYB=_arV2ylZG!m>D3Fuw!YzsjjJsiy!woV5CYb z`HBBS-nn~@{GU_JRCoPHe6}%_ju)Vju2eT=R9U0_yUEn_X}9uoHxBbyDcXonZ^b@v z*JhyiQt)Q1jp&WcL6dWz$(C)?j87@!l04W)L2J&)_<@Ew;-Vx7vZ}w7)MmsUdo}}e zwfV*Zdk~{%Iu9$v8J8%u=Qx#}*=?HH>ff6U{u&BwhO&+h5dcZ68p$argyiK}kifQv zxWGXbUv8thc9rsP&o0zD@OhoaNnNp0kuB{t_2xzK2hzRtOwW^RX62wvOoK`gC+ca( zB(3|cI*<}Q;P%~Dt=H{&U087mtc{vVuFO9V zXn|Jg8QzB8Q!^DCD&^;V=#trfN?AE>qJ`w^*tia2~!hGNM@PiS7au(K0iYdBa!)1c+^6rQK`2u^pSI2z&?N zj>)L~xBY%^u*Nty=TmiZ|Ece=NCG|18BERVjLSH%^tdo&+E2auiM5v_jy$RP%(g4d zA6dgVh|yt7nCeF>dR3W^VPzKeIeIvA;qDqWEv6+kRf_fyZPCYL!=jxpGjw4%^t%Rr z>t6cP$tPfZEtVF=FXQHX8giJOl66tp)Ly!xxb*H$xo{9-vJ+sim5e410gM%&yH=J# zDZL#KSbPSgOkiA-wgoIEHWsam4Y1#LfeHfxP6p~3xJXDyc0X5FO~EjeXE!%D0GIg) zU_qh*G4TojR>$eINbjnB-!y4a66N!7Y$l+}&m@gqb2)k?@GSAL%{Uf*`iiK{C_2HN zTkuzr4P|ZG6SL2vZ>v4`D{LnW$DR;{DvUieiv7&~WJT_%YQZi$0R4S(JPC3!?W3v} z*KA>z_{=9S)YYjOpoF8`9}BC!V}ifdj+&snML|JH74atq96ywR#m3meA{49xlWbeMom4eA>z_0? z4%Kn&9;5_7$FtInUygFr2-=tu($X0Clw2L4XFAou4x)tUX$CdVI3GM9^EutWPD^^Y zzuy6xd{N2C)0<82>hzSe&PK4Mpi`?3x=(|T`t=r9+DZ)7Vo(1mLH@)@J zc_pn}d~$H)z%5~9iS?$5?88EnlA%Qg7M2GWJ9|**)1D{ZIvE{ar-D?~t2BMLi!NOF zW79L)LY_&IGjxshUGWWR#6G1L!XOI4;G!(LJQ3bFx>6eaJlP_R<4X@K^1cJ8j-kA# zMB`FY2u)0OV=JnfK7kBa9}%LeO2bUZLRC7zweA%cC#S}KC8@#3@#*Gi-#Hn}uv)tL z{bk3^fl7{=w>lB81b+R>rq#3ESAC`#_Pbo3?&?X*&x;z4f%%asvYCC;iWx%->$CF2 z80q}pf837-t#;2?BP-ii{bn+XZ}uh|+J6oBY4Jk9O;|0nusWDjjB4hF4nJPD*}~G{ zqhSphZ+SG*p>=e9Yy1>0;kTMEQ*mTuJYHE=^?Q$l{cAUJtjf&m%@uX9B&c5f`BQHf zI!OIjoPIr*&wjSiJbuYR;212-bYEYB!xqpnvj%yAO6nV}KI0${GIZc6d;?YaKrgTP zF+EMbuJh7zTz4Uks>nSk5+5l8Cc9L_d7gubu~zCWMGYJQ0hj|9?iAqMklBD zXswlR8E9}meipn++2of)dOA6ZHR?n+-#wSfv~$EUc)V)cFxJe_&b@in(0@7;>B)7< z@Rb9}a=+wD*bpJIBo7lI@WP5^Kij7{H?q7lR4j*otFv*`g=R?gY)_*DLzLgl1Gkr* ziLJo>giK$`%#6RcmwkSvh5>|!gOig*?kIaxiiu^;1$1`G;1ZmpK?grXLe(!-)XRAH zw;5PVqaijY_HF>dETSj+in!&xoD$_m;UkRc13gz+ua?W}A`E4U3U=(75 zjqpDGN?cg@*wUBlr3f)se8|)2+4kwWw6d#QEznuYftP!W~fJqSB3(l~v~ow|rqX@)+XsaRuOmE=of`J#k7^Yhs^x3wb9&a!H>yqFWU6*cu*J+Ix2;<- z=d*p@5dT>?={|1n~=+LKB!-7DVqDz@Ru z1jE8v+NJCv)gPMlU7k{K?sR6IfA?Ke^Qw=<=W}lezfo~h zr+&63;guS?BJrP$s32v>R?ksv|2uPb{rWD|oX-&_9ev8e-@kqX;s)RZMF={Ig!%ba z$9n_nI|nK6S)M3MN(T^UL`gb1k%GmwKe#*jA(t}p<-4(MtF{=)58y~i+8xoF`_|Kg z1=iBI`1od1sqjU@ZWOb#P;6B)s;PZEzC1hlIrgZd*6{`!0$madXofg^e9nKCmYBI6 zIL^UBT+V=@NVVuO`i#a3~OTLp;Y;NaLmQm9va?A{FJ3^+-vg4S3+Mx)CH#s0xT zr@!1#-GMWApotrO9(#{p`vTJ~_8ZKDFWiwcn0iDwhZH)c4~AOz;+M6tQVE`zXCQoR z_tUeB$6F`n6#aXxOqXNJ2DW~5gz&|Z{n>F9=hLg*LwEcdA;iABdqJ7mf!sYl=dhunXL)(U zJ_BMWrd*K1YHd$aCkVM<|LEf!AJE>YnInQ0t6GAH2m9G17^{^Tw$Xv+H0ir{`Wwv? z3_{NlwH5;muu6aqi7()Fcn>%r_)vJF5`)><+11e^Tf4s?Rhb~>BLqZf6m#*<@~D#e z*smLeW*^{x0;YEdxK46|tR<+k8p9w{49*VW^DD~u0c|e=M-JbrFgC*l2;(;XED`57 zFeGKjQ-`OX{XOCsAfZVKBUksT@<^Fde^ch`v$ax)wS z+vK_wL~}m67~P2H+i^E@P0dl`0hiaWKgP$uUtN_0IZvY>2@A$0_?YOxd#M;8B>d3f z-vTM0;okB#Jn>d;g7b@-EEPs#DjFK2e}8s&c21~NLB|Dbz_5zYlM5{>p&Z_H$_H8;I@m zK$8u$EazdY71k61Y@hk%iN4g+#OtHjuYK8kgdS;J&SM)$ zsJO4&Otqxxo;bcIYLP01+8%vdIE3UuiA`UqyOI2jEw20TMTQ6H4%&1_3FBp3z`EuY=izpQ{F){g(B1{R>WnED4l!Dgy zIfxVT5+hiZ5(8m`mj6Zn=qN=9F=L#izT66LPD`CO0vFNqqEnw8@uOf2BX6X2W!XVs zbxNHP*1f@9gkn$JCH@-#QIB8yv&>QtAJ0~2FKHZe91C*IQEo*!?l?VYJ}-59zwz;& znQ$v4jTH7RSY3Z7L*;*9L?S!))gDsKzq+OQ7xHZ7xo}E_t0AOpiAG3SQTRzIuiZt) zsFvByEb_U7uWUU%rtWo|6O)t7j~)R9npjr0?@A1m@1>-r>2-ASyOA75E9o#fn*qCF ze{In(Qam*P7XAD}``OMv8f=97=>w@v5tY=(t;53v@b_V1c-;d7U5y^iHima}h6lkc zewmw(uPf&6uTt~AcyLX<59vbChY$3r;*lX?Jq3M9w@XUe@8o~Y&;JG<0?=G0C7=@^ z$yG`F4y%4)V`FMogXN1PoB)h1mHAaw!TX8tji~5a7~XHOGGrBgSdl`Lc-gnI^*38cf_mB2GM6qWv&QF%5ysK@ zs?T2Dbu9k=ToZ4C6Eiz3bU;Rf9|KBxqL=R|6ueVD>b_%)2noY_i{S1kn&RkVzGYA0 z=0+?aAYe3-Z&vGdBo1!!H$_AUBqbYk$h^@c9CnoG#NUwd@C?eSsI+&!Ux8?%sMI&X z3vm>eVEP^9eNwP;mq`{m5uW0+Ff_ykl~L5cfQG!|#%RlU%hk%A5)P6WurF0|Kjy|< zTeEaO7a&tpQ!%pRqM1wqf5V$M8BCCIO3<#5eKv$YU4L*N9lhCpaD}C$xcvYD#^RLF z4Cu^Oh=0%Z#f0&VPS36XMUUKysK6Z?;o^(C>_3o6jAsQkdd=`ekCF& z>!gYIsX~5!Sf3V!!fnc=c6j}JhpB2Ssa#)~O#1LA-7FIGN<`7F)J7(ogt{)9yLb&v z;nR>aE`o-ahURt(en`7t%hf|ntgUasgl}N&1XNcO>FIk(DioOQ362a8?{2qfl|f`> zXUF8`;ek=94RF=gzkjkiXu?E-JHNUoK?_wPlACJuOK<#rx`V(U7&GGAj@CV16e2fT!dKbR7odp)1 zQSCpmjtNAS_MhY9DM20_C5r3rI+Za|}{w|Sg|Jt8FwWRHcFI(L`j=L?Q zZc#C61S%FR0I^#n>7RW|c?`=G2SK%I92hxk*I>;@5Fj&Up8?>nv=_C>X?HN|*^@he zjQ0jr4^L*vrIYj|G&sq=9fc&3JP);RdgPXj5uyLy%CcHpUANvs#B5yY1yG zTDuQMP2oIaS3)nF_c%ZQ9?JA_d@I24JqPjc$_C-jGIX6L8q1fZjGc}o`}+FSF9gD2 z_b>ieubVjccF3k`YWOm}RL-m{&(2Q!{i>=sDL8c|v8%lHs=izt>`%IJbP$bTfXgu& z4839E;4H!k@*xgtG54K$N!pR{sJLHC94vRUs61?^F^(H&OO2ntm?A>UU5d%9ReTU* zHPzAk@>l#}LzDY9*6Ad*L`cc4DsQNQ&m#lN+vctaH~b3Eo%Wp{_-Z^76WJsz-Eyb; z_SrF+XTpk?y)14NX;@cBLmq<0UGqZb`BvWr8@Fz#`?`eGrN zZWmBl69-GcA#>81xhLDPl4-AkF^#KW79g(QOnS$|rR(I1bYaPAdM!X)N^WHk&eFuW zC$qPzq!a5(O1m?%hn(bh;HBM;u=v9-@s#EfCS?keeI zzQvATU{U`*mV>Jx&>!sUxx}TNG-rN(RUp=CMgzi zcBX=3i0ZZNtyl(;@DkMW+r~n~8u8i=<4n)XW;N?iho?NO@ki}ByN~1(Q{$OwaX|*k z@1FTT2UVp!?d}daC8h~|xfJQxO&!{cy~`f@tVKX~?f3HH16=lk>zB_Q)@`z8hSdMz%5 zBrrN)ysw!fo4T7-#dh~$@~hh0(n%%kyv&rsSGX2w#M?9HTO?QZwfwWi@p%Fb>3035 z(xM)sG(EN}Y85}x5t_;yL0j*!jn>Gt!kjSZWZH`lZagulO&T}#Q5iw*tqw_tY!%sr z#?{{>`y*2`IZif6A{gJ^MvN~)OQ4L${CJcNaqF=xUA;mv8|S^Ej!*P$p-=nc|3zqx zsS+EimsAB?JUtoreh@=%$FOCD%@?p-Vz=ldX#CZx{GZGj?g~ZR;+yApyr!!{xL`{d zre}x0Frj%lAz~MGdzQfvEv@WO^%=$r>gZa z#9xGxJQgLDFquVmc(Bl<$n6s7Qn9SY-n+Lm(ZEG&N%ftmv{mAXG&ezqCQTA;{^A9* zrobCfB5Dt_?%w;+LMaXG6zRhKP8%fZ*R66KUPz8 z4cIQxf1FwJHGErqYi@I+nU3wDg&zTPU^gk2Z(uhGJj&*{{n;;X86Gq`KURo|8R?5w z*Sg+!chN~;lZzZ;(=k1+QH`WEr{mq<_;)CpH^MI?N>ckD*WBU8kC5wXd+*ZpE+UR| z>SSKyl{H&it$F|Q;9KJ;ebqZvdba)+D-1%M*W~y|$Y^|&ahYvDpi4-Z+C(YL&{ceJ zJ}VP-hTK|b>HL6F{z2uDovhv6%$Jpa8`~!zuvY1LK6pz>X!){#O{>85?7@qTS)~A< z##4))VLrGjveC*aBMcW?xkvOGWh*o>J9Wst|BaW<;-;qlt#w@V^a5H_h9KN$LcH~( zm^dvd6?N)U9d@d1%P{pvzjsB7aC7ayx6ddudTU^vlP4pT_^}#)4K&I$Dad5INSCf3 zXvVt!(jeCzj!JlWYrW0*F^9pKFqOnhWsy*h8_xmCkWg!0(CDn{-LUO4x68Ynl&*}E zr6KL|OtfyBmD;H4pVFIIFPbdwn^-Z=>S8Ep$*qV8-k*`DF+Lc|Uz%L7lft#UPlh19 zX#yFb$*FGXz=!n%>H3HmT<8C8u>0ZD|neeV>(Fwr$9C;CM#p?2=P;^M#D2h2l* zD$tfQ&tnnuhNhi=jHjpzv)X6QYlvuGnW?OvU4czH35j8jnjKM#O(A-}D9^R$Rw+%K zI;Lv0m<)|%eMGEX*<`KRPZlNRUnmi$muKn)oE=pM#?@zz&o(Y|%H94&^O6XmG8f*J z*PLTBVD5Us&`R;$2qN5)a5`ni^Ia98UgWIn;?LChTx|dyAR1oW*mcbd{Mr+4(JV6DsS$ zf_P!bC$4dF4*IMww3FP^6Cl>D#=Ob)iW1r)r7jypc?H&nXNR0%nuQ5lD=h48^YL$m z@relx1Q;*x{8AAnp*|r?OiYAo8+Wr}s-87j+)Xib>YkeFxvxua7?$ z7Aj(5W6#ev2?N!|!PON6sE~%Hw-O`3c;FEm7njsPDv`jOb2f0j;P&|U9Kc5W2Dj~w ziHf&?exjaEP|cD1VWXpC*?4syRC;SgyuV zl~MZ}>o33+FWqK9%T~ZH?fG=$M~N+$w1y}fvdUZg>RV6+=*!di`yaC0{LKSO|GEOg z4XraX7=T~+2M1#Vav`aqLHPF4)xw10-T%dPhRO&T>$J8zgnU{TA7_VhtdY0qR9;O?9rQ#l^>)wE25D zI68Lrr+Bq4?mEo3z61md0oIa1jg70#r}L7y`}_N4&K4vL;zaELuz-OTk(SmYxP>n9 zD?PwHv7u~5gK9knK5K0V%fZ2+V?dk{MZ$Rqc)PFf_HN6S&#!VuZyZTC7OBYuy2$t0 z>U!6cp7~B)U)$^*(t+dd-VAkkt4u)p&knuDtJE(Qku$ajU*a60FC6xEQC01(k3v;I zeCF--DE0Bze_QJjr3ZE_bVRvIAwC9_zFR>B$g+`jPA7I}(csim)2zaLf&zQ{ z$1h^{HKxlm@8yP9VQ5wh;8vEGe=mFia4WQV1zgstq@?}^AeWfM#3(HK_5A$(U+ILD zY#UEEdeB2k8L+#^ibjHEUZ8ab)zX4TIW_efJi2eKegWy3nb)MHS)c+6i{<&hLy4o? zSbPs3g4qKSRYPTGXB#&eq-l_>8!R-ZuhI_!2SYuq=S-=1jz5qWzW-fzf zgr9_ON%KDwFShu5FCxIldj6LRrG)O;0YzIC&NT#{CC+m2I`^aJIFVgO0r$;h1Rv(* z7n0t*TcK^dE8!Er&APDPw}N@`h9M>212e<;KL+@2dflYuq-sgrqF>*mIkA5CE3d<>l01ZX^zB zPkHn&Bl*x=4o&PJSSgBvJZydao~zQJM>LPqAH%Xma&mHGTMNqxdOf?D3ng7$oVYj_=S&X~ zh6fLhyOB4RE)^BMq3KlE8JwGvtjA1t4XgxjLY@>AS61^AJHFj7%4mMkHgb!Vv7)5t z>HzJ~n+H>*c%Nx=Rh(yYQL#dI2=?zb-{{L zjJPykr<;h6a(I)cFn@n}TJ|0Rf9brw1yChBj;n|&Hk0)K5tYc?v`P}qsgA=$H^a*Gm`itfi6yW=Q zWIOw38)+0+T2_X_FmCl)S&d>LhbZ#ot~_w9`;iaaBW*w2I@nt1t8=c&gxOs<%~9VV z3C3a29^nD=Ik3uz+=@^ES_PQ++@5U!>>h>n{uW>!TD4|3S@7y-E6MXOsNzbl-(V6E z3-J*ZtiW6j{`6=q)YA;n+WfialOP?@w9P(1QqRejRCVlcZznq*s>Uob?>rYkz@I$1 zhX@ZHXgiFKpAi z3nzQ-2?+`EK5302X(=1p$Zwg#ynMetrler$>kr1nhlVdeCz`pAy6O%;d z=HDkj7xjDuKTEp=H+N zK3iK;# z{7W{%n--?Hv|HW~{$4cM-7&&NM}KHCoaZLAv~yoV)b|-7EZ{=Qf#G4Yfi&?h@ION} znaau`|NHqXTI?h>D(;z4At!8K2h}@*0KgdiPznuAa9WPHfMJ znLhFZz2fbaE$+3+AVS*nsI)X3TU+|t`a@&&k_YjLi9(kbN9?HZD-(pcbH^M+1Vocg zu@JD7G5jPDjA@_eojibTe|J;r1}7({_sJ3s3dIb&(bE3?JJH_C{qMc83N%ez@knWy zItI1SfW}6>os&es&3kj|)&|>Y08@d-NGW#q0|yacH>1eyeH<#=-_z+fXZsye86Jy% zMN|iB_xF-^{cAK`Sx8%j-LMzB7%?Dc;Y8_!zIHkI+2Cf7m*0H)DhSMGs_ka{Y^Tn5 zdsz88hgx52(ghpc!Wjt*^z}tR(r_bJIR#zy5vGl)DJI;3v$HeGB#BDY^Y#1wq-b>l zu9|}t;vILrSFhNzN=pewMuxQBl1_Nc99}Ky-6|H9M>9}Do1!En#ou{z&f>3sfn0PV zDQ#6>zc!llxTU-6G=LfLSXiU*q?Q;lzq9m8uRb2rmQd!r)Pdx$jJaV~kHgoMRbc0K zM(gW%+rIeJ>2`IRMYbtSiN_EhK%Yh&Y{mC%|Fy>JZA$}A+bb3E13w@fiCuYn&<_9Y z8~b!PHe@xo#s`6);Qq_TR&b|Tv1-z4;b+d2Hha&owR$74i8N3;$j5Ful zxSltntB>c+k3~kNWw`!I_|W`I2@A~-Cwoy~PR_O8L+@xrLVVm(TwB8Uzwo!5-3Zo^ z*3h^pdHWA$JwX{hT0~k~TQ7nUqj8!rSV2Q8Q5 zy{-22He20|RnzZFf7t1-(^Sc|V$aHyZC+K9{SjLvCES7U9yy?nh{tC~uEX1uK1WlO zRd3I%?9Rq=5;M{?HvZ=^TUlF6>o$0_n^m=?1q5I`v>9iL<1S|gRp3gdz{Vx zM~v(JAMNm>`a;4TFmNR9?&)cpokdam!Hc;a{4d{uK3SDkRS#|V4+(uM#4Fox_S};D z8_%B4w|V)Z?w?H9u8<>a)j@!`_Ci@uT5fO`91^KorQ^z5-t z4b*kB5h@BJd#&7qmNufI7v)U@SZ#Cpr>>3?x+mN2h;l?ps|Q%52Rw>kM6bdMep>vMQu3Mkvcs9u~)w7uN3@F`Fp}yjUn;Z?$(d z2!y6vNSJvp-X-V^J|LbM!CRZJr6cm3sJ!@n)0XhIdmT;9i3=V8YYbahiYP6@p-xqV z-2eLz3=$|A#3-&{PXzomiBvrmr2YSY|1c*3{n_u}<~{%C52|O&eCyVirh?(Mt=T5p z?dCJC>FMd8EpOj?A5NeQPboRTD9;a0L%=Lh6s80iYXp6=&fYIMImWY%9u6)pK_MYH z5cEU9ZoK-O%|39Z!L1#-z#C&F7LjUaNKIg{0TmIOn3%Y>wiabs9~v5JXkbBw4c$5v z53{0z4<*r-kU&-D40>pWC;ygKXBr-P9{v3QW_=K!ON+$ibJaC8f*`mzAA znTv>uQj3eX9G+^YfPXg?-|JOXb0GTt0I$~Z#o^2H=(Mx}0(zmC7j;gzZ-4;yt=qSG zYV7CCNpIYk-~}+p9NtO-s4Vgf4i5eRhr!RlqTvNQNSn_nWj3I&`~bVU&+uY-Q9r}Y zO)avxu!A+OG1!tL^%`71Je8L>=F+Qeh0w(dd;trj>RC-HydypkjxhsQ@(##2xwVS* zu6s4{Pu9awJvFx>Vh?o5`A}a8Jyxnmj~~`+Vq<1MP^wFuQ z{c(c8$5~umWpiC*BIMxWG6&{u55xsN6o;1<@$3vay4;ua8C>bp+s1;VnE4#$CCuUT zH+lRdTKE6XBng7Ka62~vSY9GgUk!sKHlIOkg$G=xL7^#mll=-ifzN`d0+`EXCe6*w zKR||qDmddXK06+xb<}DDo*ZA3*O56UI+`x(gTr%kSy{-gnO&5@9Z=DrzX^E${P{e% z9KvEH6BdK7hoUMk#p=KufJSu0&WSfd;_+ilK;L-ach%v3erSb7MHAJ;goHs=Rr~}5 z1mnD^sj0v7;B3xHlLHUSlrYx1N)b@9ulYp_m1p@rn)?5GRW^c-MO9v(nt zWNKy>Hsb|*;%BZ3WAz22s3)e2iwo?cQZqe${fy#b<)hp?@Caz3d};$hMKlD+^+ER% z4P2!W@P4!5?tcDM-Cz5K1G%XUwTg-gR2w%11?!Z=-s>gl1yzPY+WcHgOByat)tqd=c=C=mecGQL3o$_Cnx{Gy(l(RaMnixWVzW_my7Y#mtUSJ8L=j z-15r1pj}y949Lv9*?4g{HNWc)i1T+~v=G_b+auBs?WK!GMuGFo4LjAn>5w0$t~5)| zkZcHV3`YUFvK4}MIXae-{(%8v5Uy_rzr|707t7grO@tS8t1G{1D=Qy>KQ+VI;TC*@ zamCz|tvMm8V85&^mbiN_&=I9IHBW48?t##`KWG$`E`6EtN_4ify8501V@Yo>4k%F* zc|fya;^id;;;C4RR8@XEk~_f~&OQj)^fTp%!n z86Z@ln2Cng*6-nV3_(qO928m+R#(--I<}l!Z=Ro}R-byPB95`uR8@T;1_Gg~GXloi zXu;5Q>v_Q=jQ^7TA@(}Xx1xpp^?=0zZ~uLjkGc6BV`Jm-YBv`b8AU|`D19b)+uGXT zt@>tVnM`>{Dd$54iDGRjr-}7lI=_D10Zy<0VO*B~D$u&`dikOVM9Exnk>Gmm>FH^_ zWdLUj1fzC;+ZTCrp1=w55A}1sIDZDz5t&+{U5jN+%EOnoxmA;sBO~O@%*@7? zd3H1PSUx^JP#{@^#LGbavQY!GfGM#q411}i}UYVM5%(p83vx$tqbiTMiv$!a1|hEh7z+yNmc-L zR8dLE@#LQwL?@_xu)$vU80JrTTsO5*MhH#f#vwWB=?Hpy`m(n`ObG&t5G)kAt@|dj zva;iK&aCbt4DjL{_Lh|(_M`awfkj1}Fn|hQKmx;r3JMCa7frx)6t2NHEsYU&etJ>S zhtg8+WI<;Aq1EAcGzBM@K=zKEx>XauXo{SY-&P9A*(A*O~97H zKpb}K;_6@)Aj^JmI}oHGJ`S?m+Cp*#d^#o=PKk?)b3fi;hUuk1LFeUkShBWzl2fKd zJeKMqMD)hNMD)GD0^+{Gs%)wr8>MAoV#0DT2jOsncWG%U94_-$jlIG5iHXmjh26NK z6BF58Q)6PtKq5{s9M0YU{BmGGU`62?oAGM70lNYyqpv_nY(R<#oD z?zJ5!un#{#6lm?~!DC=xfH7%FdHHwRpCKZ8|Lb9etJtTAm9LEop|Gnqvx)g37 z+O5DSO;EFCIc$WO6B`?A!`-#A2jC#*ZZjO$sI0^{C+7~LRFVY>5p+PVNrvQsv>Cj< zZgqG9!M$z4VLY`RNOUZB?#S9?$O6a$zs7cRb3^zhCDB5Q(-G|MtBXXKzbIrd8^XH4 z&QJ}7bRukVa&i)=ju;5kO9wA>l#(<|8WF&VX*do4HF!h_Unne$U|uc(LITD8mjr;e zpTnvk!Nr6%h{i-S3TLa>`NahaS0B~2mt-UR4FQab&s;IO{U=;09dyE_1jQ0(C~$XsueULcVLlLn8C9~O$yme*Iu9V-h;`5{Q1qza2U}m zm%NEMK0cPv)un}#2FCc#(KIwPYU}Er{@{T}sMTq-&RTXekmR2{Y*+YH1H;43i7kDQ z%CjUzJ!DitYz(BA|Nig_7gS9lNW9PZ-?iHR5gV4T&~kK_?^E{ZizDElr!vZqi=_ Date: Wed, 24 Oct 2018 02:00:09 +0900 Subject: [PATCH 06/14] Fixed TypeError on creating atari vec envs (#671) --- baselines/common/cmd_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/baselines/common/cmd_util.py b/baselines/common/cmd_util.py index 44dafa1..352eda2 100644 --- a/baselines/common/cmd_util.py +++ b/baselines/common/cmd_util.py @@ -32,7 +32,8 @@ def make_vec_env(env_id, env_type, num_env, seed, wrapper_kwargs=None, start_ind subrank = rank, seed=seed, reward_scale=reward_scale, - gamestate=gamestate + gamestate=gamestate, + wrapper_kwargs=wrapper_kwargs ) set_global_seeds(seed) From 014a5597b1e93ef54d9593e78ab9d094f844a36d Mon Sep 17 00:00:00 2001 From: pzhokhov Date: Tue, 23 Oct 2018 10:01:25 -0700 Subject: [PATCH 07/14] refactor ACER (#664) * make acer use vecframestack * acer passes mnist test with 20k steps * acer with non-image observations and tests * flake8 * test acer serialization with non-recurrent policies --- baselines/acer/acer.py | 26 +++--- baselines/acer/buffer.py | 93 +++++++++++++++----- baselines/acer/runner.py | 41 ++++----- baselines/common/tests/test_cartpole.py | 3 +- baselines/common/tests/test_identity.py | 4 +- baselines/common/tests/test_mnist.py | 5 +- baselines/common/tests/test_serialization.py | 3 +- baselines/run.py | 4 +- 8 files changed, 118 insertions(+), 61 deletions(-) diff --git a/baselines/acer/acer.py b/baselines/acer/acer.py index 4e2e00f..0ae0330 100644 --- a/baselines/acer/acer.py +++ b/baselines/acer/acer.py @@ -7,6 +7,7 @@ from baselines import logger from baselines.common import set_global_seeds from baselines.common.policies import build_policy from baselines.common.tf_util import get_session, save_variables +from baselines.common.vec_env.vec_frame_stack import VecFrameStack from baselines.a2c.utils import batch_to_seq, seq_to_batch from baselines.a2c.utils import cat_entropy_softmax @@ -55,8 +56,7 @@ def q_retrace(R, D, q_i, v, rho_i, nenvs, nsteps, gamma): # return tf.minimum(1 + eps_clip, tf.maximum(1 - eps_clip, ratio)) class Model(object): - def __init__(self, policy, ob_space, ac_space, nenvs, nsteps, nstack, num_procs, - ent_coef, q_coef, gamma, max_grad_norm, lr, + def __init__(self, policy, ob_space, ac_space, nenvs, nsteps, ent_coef, q_coef, gamma, max_grad_norm, lr, rprop_alpha, rprop_epsilon, total_timesteps, lrschedule, c, trust_region, alpha, delta): @@ -71,8 +71,8 @@ class Model(object): LR = tf.placeholder(tf.float32, []) eps = 1e-6 - step_ob_placeholder = tf.placeholder(dtype=ob_space.dtype, shape=(nenvs,) + ob_space.shape[:-1] + (ob_space.shape[-1] * nstack,)) - train_ob_placeholder = tf.placeholder(dtype=ob_space.dtype, shape=(nenvs*(nsteps+1),) + ob_space.shape[:-1] + (ob_space.shape[-1] * nstack,)) + step_ob_placeholder = tf.placeholder(dtype=ob_space.dtype, shape=(nenvs,) + ob_space.shape) + train_ob_placeholder = tf.placeholder(dtype=ob_space.dtype, shape=(nenvs*(nsteps+1),) + ob_space.shape) with tf.variable_scope('acer_model', reuse=tf.AUTO_REUSE): step_model = policy(observ_placeholder=step_ob_placeholder, sess=sess) @@ -247,6 +247,7 @@ class Acer(): # get obs, actions, rewards, mus, dones from buffer. obs, actions, rewards, mus, dones, masks = buffer.get() + # reshape stuff correctly obs = obs.reshape(runner.batch_ob_shape) actions = actions.reshape([runner.nbatch]) @@ -270,7 +271,7 @@ class Acer(): logger.dump_tabular() -def learn(network, env, seed=None, nsteps=20, nstack=4, total_timesteps=int(80e6), q_coef=0.5, ent_coef=0.01, +def learn(network, env, seed=None, nsteps=20, total_timesteps=int(80e6), q_coef=0.5, ent_coef=0.01, max_grad_norm=10, lr=7e-4, lrschedule='linear', rprop_epsilon=1e-5, rprop_alpha=0.99, gamma=0.99, log_interval=100, buffer_size=50000, replay_ratio=4, replay_start=10000, c=10.0, trust_region=True, alpha=0.99, delta=1, load_path=None, **network_kwargs): @@ -342,21 +343,24 @@ def learn(network, env, seed=None, nsteps=20, nstack=4, total_timesteps=int(80e6 print("Running Acer Simple") print(locals()) set_global_seeds(seed) - policy = build_policy(env, network, estimate_q=True, **network_kwargs) + if not isinstance(env, VecFrameStack): + env = VecFrameStack(env, 1) + policy = build_policy(env, network, estimate_q=True, **network_kwargs) nenvs = env.num_envs ob_space = env.observation_space ac_space = env.action_space - num_procs = len(env.remotes) if hasattr(env, 'remotes') else 1# HACK - model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps, nstack=nstack, - num_procs=num_procs, ent_coef=ent_coef, q_coef=q_coef, gamma=gamma, + + nstack = env.nstack + model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps, + ent_coef=ent_coef, q_coef=q_coef, gamma=gamma, max_grad_norm=max_grad_norm, lr=lr, rprop_alpha=rprop_alpha, rprop_epsilon=rprop_epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule, c=c, trust_region=trust_region, alpha=alpha, delta=delta) - runner = Runner(env=env, model=model, nsteps=nsteps, nstack=nstack) + runner = Runner(env=env, model=model, nsteps=nsteps) if replay_ratio > 0: - buffer = Buffer(env=env, nsteps=nsteps, nstack=nstack, size=buffer_size) + buffer = Buffer(env=env, nsteps=nsteps, size=buffer_size) else: buffer = None nbatch = nenvs*nsteps diff --git a/baselines/acer/buffer.py b/baselines/acer/buffer.py index 2dcfa10..000592c 100644 --- a/baselines/acer/buffer.py +++ b/baselines/acer/buffer.py @@ -2,11 +2,16 @@ import numpy as np class Buffer(object): # gets obs, actions, rewards, mu's, (states, masks), dones - def __init__(self, env, nsteps, nstack, size=50000): + def __init__(self, env, nsteps, size=50000): self.nenv = env.num_envs self.nsteps = nsteps - self.nh, self.nw, self.nc = env.observation_space.shape - self.nstack = nstack + # self.nh, self.nw, self.nc = env.observation_space.shape + self.obs_shape = env.observation_space.shape + self.obs_dtype = env.observation_space.dtype + self.ac_dtype = env.action_space.dtype + self.nc = self.obs_shape[-1] + self.nstack = env.nstack + self.nc //= self.nstack self.nbatch = self.nenv * self.nsteps self.size = size // (self.nsteps) # Each loc contains nenv * nsteps frames, thus total buffer is nenv * size frames @@ -33,22 +38,11 @@ class Buffer(object): # Generate stacked frames def decode(self, enc_obs, dones): # enc_obs has shape [nenvs, nsteps + nstack, nh, nw, nc] - # dones has shape [nenvs, nsteps, nh, nw, nc] + # dones has shape [nenvs, nsteps] # returns stacked obs of shape [nenv, (nsteps + 1), nh, nw, nstack*nc] - nstack, nenv, nsteps, nh, nw, nc = self.nstack, self.nenv, self.nsteps, self.nh, self.nw, self.nc - y = np.empty([nsteps + nstack - 1, nenv, 1, 1, 1], dtype=np.float32) - obs = np.zeros([nstack, nsteps + nstack, nenv, nh, nw, nc], dtype=np.uint8) - x = np.reshape(enc_obs, [nenv, nsteps + nstack, nh, nw, nc]).swapaxes(1, - 0) # [nsteps + nstack, nenv, nh, nw, nc] - y[3:] = np.reshape(1.0 - dones, [nenv, nsteps, 1, 1, 1]).swapaxes(1, 0) # keep - y[:3] = 1.0 - # y = np.reshape(1 - dones, [nenvs, nsteps, 1, 1, 1]) - for i in range(nstack): - obs[-(i + 1), i:] = x - # obs[:,i:,:,:,-(i+1),:] = x - x = x[:-1] * y - y = y[1:] - return np.reshape(obs[:, 3:].transpose((2, 1, 3, 4, 0, 5)), [nenv, (nsteps + 1), nh, nw, nstack * nc]) + + return _stack_obs(enc_obs, dones, + nsteps=self.nsteps) def put(self, enc_obs, actions, rewards, mus, dones, masks): # enc_obs [nenv, (nsteps + nstack), nh, nw, nc] @@ -56,8 +50,8 @@ class Buffer(object): # mus [nenv, nsteps, nact] if self.enc_obs is None: - self.enc_obs = np.empty([self.size] + list(enc_obs.shape), dtype=np.uint8) - self.actions = np.empty([self.size] + list(actions.shape), dtype=np.int32) + self.enc_obs = np.empty([self.size] + list(enc_obs.shape), dtype=self.obs_dtype) + self.actions = np.empty([self.size] + list(actions.shape), dtype=self.ac_dtype) self.rewards = np.empty([self.size] + list(rewards.shape), dtype=np.float32) self.mus = np.empty([self.size] + list(mus.shape), dtype=np.float32) self.dones = np.empty([self.size] + list(dones.shape), dtype=np.bool) @@ -101,3 +95,62 @@ class Buffer(object): mus = take(self.mus) masks = take(self.masks) return obs, actions, rewards, mus, dones, masks + + + +def _stack_obs_ref(enc_obs, dones, nsteps): + nenv = enc_obs.shape[0] + nstack = enc_obs.shape[1] - nsteps + nh, nw, nc = enc_obs.shape[2:] + obs_dtype = enc_obs.dtype + obs_shape = (nh, nw, nc*nstack) + + mask = np.empty([nsteps + nstack - 1, nenv, 1, 1, 1], dtype=np.float32) + obs = np.zeros([nstack, nsteps + nstack, nenv, nh, nw, nc], dtype=obs_dtype) + x = np.reshape(enc_obs, [nenv, nsteps + nstack, nh, nw, nc]).swapaxes(1, 0) # [nsteps + nstack, nenv, nh, nw, nc] + + mask[nstack-1:] = np.reshape(1.0 - dones, [nenv, nsteps, 1, 1, 1]).swapaxes(1, 0) # keep + mask[:nstack-1] = 1.0 + + # y = np.reshape(1 - dones, [nenvs, nsteps, 1, 1, 1]) + for i in range(nstack): + obs[-(i + 1), i:] = x + # obs[:,i:,:,:,-(i+1),:] = x + x = x[:-1] * mask + mask = mask[1:] + + return np.reshape(obs[:, (nstack-1):].transpose((2, 1, 3, 4, 0, 5)), (nenv, (nsteps + 1)) + obs_shape) + +def _stack_obs(enc_obs, dones, nsteps): + nenv = enc_obs.shape[0] + nstack = enc_obs.shape[1] - nsteps + nc = enc_obs.shape[-1] + + obs_ = np.zeros((nenv, nsteps + 1) + enc_obs.shape[2:-1] + (enc_obs.shape[-1] * nstack, ), dtype=enc_obs.dtype) + mask = np.ones((nenv, nsteps+1), dtype=enc_obs.dtype) + mask[:, 1:] = 1.0 - dones + mask = mask.reshape(mask.shape + tuple(np.ones(len(enc_obs.shape)-2, dtype=np.uint8))) + + for i in range(nstack-1, -1, -1): + obs_[..., i * nc : (i + 1) * nc] = enc_obs[:, i : i + nsteps + 1, :] + if i < nstack-1: + obs_[..., i * nc : (i + 1) * nc] *= mask + mask[:, 1:, ...] *= mask[:, :-1, ...] + + return obs_ + +def test_stack_obs(): + nstack = 7 + nenv = 1 + nsteps = 5 + + obs_shape = (2, 3, nstack) + + enc_obs_shape = (nenv, nsteps + nstack) + obs_shape[:-1] + (1,) + enc_obs = np.random.random(enc_obs_shape) + dones = np.random.randint(low=0, high=2, size=(nenv, nsteps)) + + stacked_obs_ref = _stack_obs_ref(enc_obs, dones, nsteps=nsteps) + stacked_obs_test = _stack_obs(enc_obs, dones, nsteps=nsteps) + + np.testing.assert_allclose(stacked_obs_ref, stacked_obs_test) diff --git a/baselines/acer/runner.py b/baselines/acer/runner.py index 6bc1b4c..afd19ce 100644 --- a/baselines/acer/runner.py +++ b/baselines/acer/runner.py @@ -1,30 +1,31 @@ import numpy as np from baselines.common.runners import AbstractEnvRunner +from baselines.common.vec_env.vec_frame_stack import VecFrameStack +from gym import spaces + class Runner(AbstractEnvRunner): - def __init__(self, env, model, nsteps, nstack): + def __init__(self, env, model, nsteps): super().__init__(env=env, model=model, nsteps=nsteps) - self.nstack = nstack - nh, nw, nc = env.observation_space.shape - self.nc = nc # nc = 1 for atari, but just in case + assert isinstance(env.action_space, spaces.Discrete), 'This ACER implementation works only with discrete action spaces!' + assert isinstance(env, VecFrameStack) + self.nact = env.action_space.n nenv = self.nenv self.nbatch = nenv * nsteps - self.batch_ob_shape = (nenv*(nsteps+1), nh, nw, nc*nstack) - self.obs = np.zeros((nenv, nh, nw, nc * nstack), dtype=np.uint8) - obs = env.reset() - self.update_obs(obs) + self.batch_ob_shape = (nenv*(nsteps+1),) + env.observation_space.shape + + self.obs = env.reset() + self.obs_dtype = env.observation_space.dtype + self.ac_dtype = env.action_space.dtype + self.nstack = self.env.nstack + self.nc = self.batch_ob_shape[-1] // self.nstack - def update_obs(self, obs, dones=None): - #self.obs = obs - if dones is not None: - self.obs *= (1 - dones.astype(np.uint8))[:, None, None, None] - self.obs = np.roll(self.obs, shift=-self.nc, axis=3) - self.obs[:, :, :, -self.nc:] = obs[:, :, :, :] def run(self): - enc_obs = np.split(self.obs, self.nstack, axis=3) # so now list of obs steps + # enc_obs = np.split(self.obs, self.nstack, axis=3) # so now list of obs steps + enc_obs = np.split(self.env.stackedobs, self.env.nstack, axis=-1) mb_obs, mb_actions, mb_mus, mb_dones, mb_rewards = [], [], [], [], [] for _ in range(self.nsteps): actions, mus, states = self.model._step(self.obs, S=self.states, M=self.dones) @@ -36,15 +37,15 @@ class Runner(AbstractEnvRunner): # states information for statefull models like LSTM self.states = states self.dones = dones - self.update_obs(obs, dones) + self.obs = obs mb_rewards.append(rewards) - enc_obs.append(obs) + enc_obs.append(obs[..., -self.nc:]) mb_obs.append(np.copy(self.obs)) mb_dones.append(self.dones) - enc_obs = np.asarray(enc_obs, dtype=np.uint8).swapaxes(1, 0) - mb_obs = np.asarray(mb_obs, dtype=np.uint8).swapaxes(1, 0) - mb_actions = np.asarray(mb_actions, dtype=np.int32).swapaxes(1, 0) + enc_obs = np.asarray(enc_obs, dtype=self.obs_dtype).swapaxes(1, 0) + mb_obs = np.asarray(mb_obs, dtype=self.obs_dtype).swapaxes(1, 0) + mb_actions = np.asarray(mb_actions, dtype=self.ac_dtype).swapaxes(1, 0) mb_rewards = np.asarray(mb_rewards, dtype=np.float32).swapaxes(1, 0) mb_mus = np.asarray(mb_mus, dtype=np.float32).swapaxes(1, 0) diff --git a/baselines/common/tests/test_cartpole.py b/baselines/common/tests/test_cartpole.py index 06d65e4..475ad1d 100644 --- a/baselines/common/tests/test_cartpole.py +++ b/baselines/common/tests/test_cartpole.py @@ -13,6 +13,7 @@ common_kwargs = dict( learn_kwargs = { 'a2c' : dict(nsteps=32, value_network='copy', lr=0.05), + 'acer': dict(value_network='copy'), 'acktr': dict(nsteps=32, value_network='copy', is_async=False), 'deepq': dict(total_timesteps=20000), 'ppo2': dict(value_network='copy'), @@ -40,4 +41,4 @@ def test_cartpole(alg): reward_per_episode_test(env_fn, learn_fn, 100) if __name__ == '__main__': - test_cartpole('deepq') + test_cartpole('acer') diff --git a/baselines/common/tests/test_identity.py b/baselines/common/tests/test_identity.py index 744ed83..0b3c46e 100644 --- a/baselines/common/tests/test_identity.py +++ b/baselines/common/tests/test_identity.py @@ -20,8 +20,8 @@ learn_kwargs = { } -algos_disc = ['a2c', 'deepq', 'ppo2', 'trpo_mpi'] -algos_cont = ['a2c', 'ddpg', 'ppo2', 'trpo_mpi'] +algos_disc = ['a2c', 'acktr', 'deepq', 'ppo2', 'trpo_mpi'] +algos_cont = ['a2c', 'acktr', 'ddpg', 'ppo2', 'trpo_mpi'] @pytest.mark.slow @pytest.mark.parametrize("alg", algos_disc) diff --git a/baselines/common/tests/test_mnist.py b/baselines/common/tests/test_mnist.py index 536164f..eea094d 100644 --- a/baselines/common/tests/test_mnist.py +++ b/baselines/common/tests/test_mnist.py @@ -17,8 +17,7 @@ common_kwargs = { learn_args = { 'a2c': dict(total_timesteps=50000), - # TODO need to resolve inference (step) API differences for acer; also slow - # 'acer': dict(seed=0, total_timesteps=1000), + 'acer': dict(total_timesteps=20000), 'deepq': dict(total_timesteps=5000), 'acktr': dict(total_timesteps=30000), 'ppo2': dict(total_timesteps=50000, lr=1e-3, nsteps=128, ent_coef=0.0), @@ -47,4 +46,4 @@ def test_mnist(alg): simple_test(env_fn, learn_fn, 0.6) if __name__ == '__main__': - test_mnist('deepq') + test_mnist('acer') diff --git a/baselines/common/tests/test_serialization.py b/baselines/common/tests/test_serialization.py index f46b578..fac4929 100644 --- a/baselines/common/tests/test_serialization.py +++ b/baselines/common/tests/test_serialization.py @@ -17,6 +17,7 @@ learn_kwargs = { 'deepq': {}, 'a2c': {}, 'acktr': {}, + 'acer': {}, 'ppo2': {'nminibatches': 1, 'nsteps': 10}, 'trpo_mpi': {}, } @@ -37,7 +38,7 @@ def test_serialization(learn_fn, network_fn): ''' - if network_fn.endswith('lstm') and learn_fn in ['acktr', 'trpo_mpi', 'deepq']: + if network_fn.endswith('lstm') and learn_fn in ['acer', 'acktr', 'trpo_mpi', 'deepq']: # TODO make acktr work with recurrent policies # and test # github issue: https://github.com/openai/baselines/issues/660 diff --git a/baselines/run.py b/baselines/run.py index dedca8b..4aaf1a7 100644 --- a/baselines/run.py +++ b/baselines/run.py @@ -91,9 +91,7 @@ def build_env(args): env_type, env_id = get_env_type(args.env) if env_type in {'atari', 'retro'}: - if alg == 'acer': - env = make_vec_env(env_id, env_type, nenv, seed) - elif alg == 'deepq': + if alg == 'deepq': env = make_env(env_id, env_type, seed=seed, wrapper_kwargs={'frame_stack': True}) elif alg == 'trpo_mpi': env = make_env(env_id, env_type, seed=seed) From 583ba082a2ade49455030f38374e889874b885fd Mon Sep 17 00:00:00 2001 From: pzhokhov Date: Tue, 23 Oct 2018 11:22:27 -0700 Subject: [PATCH 08/14] Update cmd_util.py --- baselines/common/cmd_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/common/cmd_util.py b/baselines/common/cmd_util.py index 352eda2..7c38a77 100644 --- a/baselines/common/cmd_util.py +++ b/baselines/common/cmd_util.py @@ -43,7 +43,7 @@ def make_vec_env(env_id, env_type, num_env, seed, wrapper_kwargs=None, start_ind return DummyVecEnv([make_thunk(start_index)]) -def make_env(env_id, env_type, subrank=0, seed=None, reward_scale=1.0, gamestate=None, wrapper_kwargs=None): +def make_env(env_id, env_type, subrank=0, seed=None, reward_scale=1.0, gamestate=None, wrapper_kwargs={}): mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0 if env_type == 'atari': env = make_atari(env_id) From 88300ed54c55d91cd0ba2954fcdfc7fd1c47929f Mon Sep 17 00:00:00 2001 From: Peter Zhokhov Date: Wed, 24 Oct 2018 09:57:57 -0700 Subject: [PATCH 09/14] fix raise NotImplemented() complaints of latest flake8 --- baselines/deepq/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/baselines/deepq/utils.py b/baselines/deepq/utils.py index 0fb1569..5176f32 100644 --- a/baselines/deepq/utils.py +++ b/baselines/deepq/utils.py @@ -18,11 +18,11 @@ class TfInput(object): """Return the tf variable(s) representing the possibly postprocessed value of placeholder(s). """ - raise NotImplemented() + raise NotImplementedError def make_feed_dict(data): """Given data input it to the placeholder(s).""" - raise NotImplemented() + raise NotImplementedError class PlaceholderTfInput(TfInput): From 84ea7aa1fd83966d6a121e60e14dfaac4675c767 Mon Sep 17 00:00:00 2001 From: AurelianTactics Date: Wed, 24 Oct 2018 12:59:46 -0400 Subject: [PATCH 10/14] DDPG has unused 'seed' argument (#676) DeepQ, PPO2, ACER, trpo_mpi, A2C, and ACKTR have the code for: ``` from baselines.common import set_global_seeds ... def learn(...): ... set_global_seeds(seed) ``` DDPG has the argument 'seed=None' but doesn't have the two lines of code needed to set the global seeds. --- baselines/ddpg/ddpg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/baselines/ddpg/ddpg.py b/baselines/ddpg/ddpg.py index 181f923..8b8659b 100755 --- a/baselines/ddpg/ddpg.py +++ b/baselines/ddpg/ddpg.py @@ -7,7 +7,7 @@ from baselines.ddpg.ddpg_learner import DDPG from baselines.ddpg.models import Actor, Critic from baselines.ddpg.memory import Memory from baselines.ddpg.noise import AdaptiveParamNoiseSpec, NormalActionNoise, OrnsteinUhlenbeckActionNoise - +from baselines.common import set_global_seeds import baselines.common.tf_util as U from baselines import logger @@ -41,6 +41,7 @@ def learn(network, env, param_noise_adaption_interval=50, **network_kwargs): + set_global_seeds(seed) if total_timesteps is not None: assert nb_epochs is None From c3bd8cea665afa8de6ac3eccc870e5952938c8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juliano=20Lagan=C3=A1?= Date: Wed, 24 Oct 2018 19:00:31 +0200 Subject: [PATCH 11/14] Adds description of param_noise parameter in deepq.learn method (#675) --- baselines/deepq/deepq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/baselines/deepq/deepq.py b/baselines/deepq/deepq.py index c8de92d..b7b9d1a 100644 --- a/baselines/deepq/deepq.py +++ b/baselines/deepq/deepq.py @@ -169,6 +169,8 @@ def learn(env, to 1.0. If set to None equals to total_timesteps. prioritized_replay_eps: float epsilon to add to the TD errors when updating priorities. + param_noise: bool + whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905) callback: (locals, globals) -> None function called at every steps with state of the algorithm. If callback returns true training stops. From 8e56ddeac296deab3cc0adc79c84a5abb59d7a3a Mon Sep 17 00:00:00 2001 From: pzhokhov Date: Wed, 24 Oct 2018 11:01:59 -0700 Subject: [PATCH 12/14] Multidiscrete action space compatibility for policy gradient-based methods (#677) * multidiscrete space compatibility * flake8 and syntax --- baselines/acktr/acktr.py | 10 +++++----- baselines/common/distributions.py | 7 ++++++- baselines/common/input.py | 16 ++++++++++++---- baselines/common/tests/envs/identity_env.py | 15 ++++++++++++++- baselines/common/tests/test_identity.py | 20 ++++++++++++++++++-- baselines/common/vec_env/dummy_vec_env.py | 3 +++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/baselines/acktr/acktr.py b/baselines/acktr/acktr.py index dcbe612..10ab32b 100644 --- a/baselines/acktr/acktr.py +++ b/baselines/acktr/acktr.py @@ -21,16 +21,16 @@ class Model(object): self.sess = sess = get_session() nbatch = nenvs * nsteps - A = tf.placeholder(ac_space.dtype, [nbatch,] + list(ac_space.shape)) + with tf.variable_scope('acktr_model', reuse=tf.AUTO_REUSE): + self.model = step_model = policy(nenvs, 1, sess=sess) + self.model2 = train_model = policy(nenvs*nsteps, nsteps, sess=sess) + + A = train_model.pdtype.sample_placeholder([None]) ADV = tf.placeholder(tf.float32, [nbatch]) R = tf.placeholder(tf.float32, [nbatch]) PG_LR = tf.placeholder(tf.float32, []) VF_LR = tf.placeholder(tf.float32, []) - with tf.variable_scope('acktr_model', reuse=tf.AUTO_REUSE): - self.model = step_model = policy(nenvs, 1, sess=sess) - self.model2 = train_model = policy(nenvs*nsteps, nsteps, sess=sess) - neglogpac = train_model.pd.neglogp(A) self.logits = train_model.pi diff --git a/baselines/common/distributions.py b/baselines/common/distributions.py index 491b9ff..5b3e7be 100644 --- a/baselines/common/distributions.py +++ b/baselines/common/distributions.py @@ -39,7 +39,7 @@ class PdType(object): raise NotImplementedError def pdfromflat(self, flat): return self.pdclass()(flat) - def pdfromlatent(self, latent_vector): + def pdfromlatent(self, latent_vector, init_scale, init_bias): raise NotImplementedError def param_shape(self): raise NotImplementedError @@ -80,6 +80,11 @@ class MultiCategoricalPdType(PdType): return MultiCategoricalPd def pdfromflat(self, flat): return MultiCategoricalPd(self.ncats, flat) + + def pdfromlatent(self, latent, init_scale=1.0, init_bias=0.0): + pdparam = fc(latent, 'pi', self.ncats.sum(), init_scale=init_scale, init_bias=init_bias) + return self.pdfromflat(pdparam), pdparam + def param_shape(self): return [sum(self.ncats)] def sample_shape(self): diff --git a/baselines/common/input.py b/baselines/common/input.py index 7d51008..ebaf30a 100644 --- a/baselines/common/input.py +++ b/baselines/common/input.py @@ -1,5 +1,6 @@ +import numpy as np import tensorflow as tf -from gym.spaces import Discrete, Box +from gym.spaces import Discrete, Box, MultiDiscrete def observation_placeholder(ob_space, batch_size=None, name='Ob'): ''' @@ -20,10 +21,14 @@ def observation_placeholder(ob_space, batch_size=None, name='Ob'): tensorflow placeholder tensor ''' - assert isinstance(ob_space, Discrete) or isinstance(ob_space, Box), \ + assert isinstance(ob_space, Discrete) or isinstance(ob_space, Box) or isinstance(ob_space, MultiDiscrete), \ 'Can only deal with Discrete and Box observation spaces for now' - return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=ob_space.dtype, name=name) + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) def observation_input(ob_space, batch_size=None, name='Ob'): @@ -48,9 +53,12 @@ def encode_observation(ob_space, placeholder): ''' if isinstance(ob_space, Discrete): return tf.to_float(tf.one_hot(placeholder, ob_space.n)) - elif isinstance(ob_space, Box): return tf.to_float(placeholder) + elif isinstance(ob_space, MultiDiscrete): + placeholder = tf.cast(placeholder, tf.int32) + one_hots = [tf.to_float(tf.one_hot(placeholder[..., i], ob_space.nvec[i])) for i in range(placeholder.shape[-1])] + return tf.concat(one_hots, axis=-1) else: raise NotImplementedError diff --git a/baselines/common/tests/envs/identity_env.py b/baselines/common/tests/envs/identity_env.py index 005d3ff..4429f04 100644 --- a/baselines/common/tests/envs/identity_env.py +++ b/baselines/common/tests/envs/identity_env.py @@ -1,7 +1,7 @@ import numpy as np from abc import abstractmethod from gym import Env -from gym.spaces import Discrete, Box +from gym.spaces import MultiDiscrete, Discrete, Box class IdentityEnv(Env): @@ -53,6 +53,19 @@ class DiscreteIdentityEnv(IdentityEnv): def _get_reward(self, actions): return 1 if self.state == actions else 0 +class MultiDiscreteIdentityEnv(IdentityEnv): + def __init__( + self, + dims, + episode_len=None, + ): + + self.action_space = MultiDiscrete(dims) + super().__init__(episode_len=episode_len) + + def _get_reward(self, actions): + return 1 if all(self.state == actions) else 0 + class BoxIdentityEnv(IdentityEnv): def __init__( diff --git a/baselines/common/tests/test_identity.py b/baselines/common/tests/test_identity.py index 0b3c46e..c950e5a 100644 --- a/baselines/common/tests/test_identity.py +++ b/baselines/common/tests/test_identity.py @@ -1,5 +1,5 @@ import pytest -from baselines.common.tests.envs.identity_env import DiscreteIdentityEnv, BoxIdentityEnv +from baselines.common.tests.envs.identity_env import DiscreteIdentityEnv, BoxIdentityEnv, MultiDiscreteIdentityEnv from baselines.run import get_learn_function from baselines.common.tests.util import simple_test @@ -21,6 +21,7 @@ learn_kwargs = { algos_disc = ['a2c', 'acktr', 'deepq', 'ppo2', 'trpo_mpi'] +algos_multidisc = ['a2c', 'acktr', 'ppo2', 'trpo_mpi'] algos_cont = ['a2c', 'acktr', 'ddpg', 'ppo2', 'trpo_mpi'] @pytest.mark.slow @@ -38,6 +39,21 @@ def test_discrete_identity(alg): env_fn = lambda: DiscreteIdentityEnv(10, episode_len=100) simple_test(env_fn, learn_fn, 0.9) +@pytest.mark.slow +@pytest.mark.parametrize("alg", algos_multidisc) +def test_multidiscrete_identity(alg): + ''' + Test if the algorithm (with an mlp policy) + can learn an identity transformation (i.e. return observation as an action) + ''' + + kwargs = learn_kwargs[alg] + kwargs.update(common_kwargs) + + learn_fn = lambda e: get_learn_function(alg)(env=e, **kwargs) + env_fn = lambda: MultiDiscreteIdentityEnv((3,3), episode_len=100) + simple_test(env_fn, learn_fn, 0.9) + @pytest.mark.slow @pytest.mark.parametrize("alg", algos_cont) def test_continuous_identity(alg): @@ -55,5 +71,5 @@ def test_continuous_identity(alg): simple_test(env_fn, learn_fn, -0.1) if __name__ == '__main__': - test_continuous_identity('ddpg') + test_multidiscrete_identity('acktr') diff --git a/baselines/common/vec_env/dummy_vec_env.py b/baselines/common/vec_env/dummy_vec_env.py index 60db11d..2b4d2ba 100644 --- a/baselines/common/vec_env/dummy_vec_env.py +++ b/baselines/common/vec_env/dummy_vec_env.py @@ -20,8 +20,11 @@ class DummyVecEnv(VecEnv): env = self.envs[0] VecEnv.__init__(self, len(env_fns), env.observation_space, env.action_space) obs_space = env.observation_space + if isinstance(obs_space, spaces.MultiDiscrete): + obs_space.shape = obs_space.shape[0] self.keys, shapes, dtypes = obs_space_info(obs_space) + self.buf_obs = { k: np.zeros((self.num_envs,) + tuple(shapes[k]), dtype=dtypes[k]) for k in self.keys } self.buf_dones = np.zeros((self.num_envs,), dtype=np.bool) self.buf_rews = np.zeros((self.num_envs,), dtype=np.float32) From e2b41828af969f2b0c6450a53b8827946ff8673d Mon Sep 17 00:00:00 2001 From: Mathieu Poliquin Date: Tue, 30 Oct 2018 04:30:41 +0800 Subject: [PATCH 13/14] Set 'cnn' as default network for retro (#683) --- baselines/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/run.py b/baselines/run.py index 4aaf1a7..28cf620 100644 --- a/baselines/run.py +++ b/baselines/run.py @@ -131,7 +131,7 @@ def get_env_type(env_id): def get_default_network(env_type): - if env_type == 'atari': + if env_type in {'atari', 'retro'}: return 'cnn' else: return 'mlp' From de36116e3bec9610932fd3034fc4bc7fc8271f74 Mon Sep 17 00:00:00 2001 From: Peter Zhokhov Date: Mon, 29 Oct 2018 15:25:31 -0700 Subject: [PATCH 14/14] update tensorflow version check regex to parse version like 1.2.3rc4 (previously only 1.2.3-rc4) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 425a1e8..7244c18 100644 --- a/setup.py +++ b/setup.py @@ -57,4 +57,4 @@ for tf_pkg_name in ['tensorflow', 'tensorflow-gpu']: pass assert tf_pkg is not None, 'TensorFlow needed, of version above 1.4' from distutils.version import StrictVersion -assert StrictVersion(re.sub(r'-rc\d+$', '', tf_pkg.version)) >= StrictVersion('1.4.0') +assert StrictVersion(re.sub(r'-?rc\d+$', '', tf_pkg.version)) >= StrictVersion('1.4.0')