gaspersic/Gymnasium
1
0
Fork 0
mirror of https://github.com/Farama-Foundation/Gymnasium.git synced 2025-08-19 05:25:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Monitored wrapper (#434)

Browse Source 
* Add WIP Monitored wrapper

* Remove irrelevant render after close monitor test

* py27 compatibility

* Fix test_benchmark

* Move Monitored out of wrappers __init__

* Turn Monitored into a function that returns a Monitor class

* Fix monitor tests

* Remove deprecated test

* Remove deprecated utility

* Prevent duplicate wrapping, add test

* Fix test

* close env in tests to prevent writing to nonexistent file

* Disable semisuper tests

* typo

* Fix failing spec

* Fix monitoring on semisuper tasks

* Allow disabling of duplicate check

* Rename MonitorManager

* Monitored -> Monitor

* Clean up comments

* Remove cruft
This commit is contained in:
Tom Brown
2016-12-23 16:21:42 -08:00
committed by GitHub
parent dc07c7d414
commit 2d44ed4968
18 changed files with 176 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
examples/agents/random_agent.py
View File

@@ -1,12 +1,13 @@
import argparse
import logging
import os
import sys
import gym
from gym import wrappers
# The world's simplest agent!
class RandomAgent(object):
"""The world's simplest agent!"""
def __init__(self, action_space):
self.action_space = action_space
@@ -39,12 +40,8 @@ if __name__ == '__main__':
# will be namespaced). You can also dump to a tempdir if you'd
# like: tempfile.mkdtemp().
outdir = '/tmp/random-agent-results'
env = wrappers.Monitor(directory=outdir, force=True)(env)
env.seed(0)
env.monitor.start(outdir, force=True)
# This declaration must go *after* the monitor call, since the
# monitor's seeding creates a new action_space instance with the
# appropriate pseudorandom number generator.
agent = RandomAgent(env.action_space)
episode_count = 100
@@ -62,8 +59,8 @@ if __name__ == '__main__':
# render if asked by env.monitor: it calls env.render('rgb_array') to record video.
# Video is not recorded every episode, see capped_cubic_video_schedule for details.
# Dump result info to disk
env.monitor.close()
# Close the env and write monitor result info to disk
env.close()
# Upload to the scoreboard. We could also do this from another
# process if we wanted.

8
examples/scripts/benchmark_runner
View File

@@ -13,6 +13,8 @@ import sys
import gym
# In modules, use `logger = logging.getLogger(__name__)`
from gym import wrappers
logger = logging.getLogger()
def main():
@@ -41,16 +43,16 @@ def main():
# run benchmark tasks
for task in benchmark.tasks:
logger.info("Running on env: {}".format(task.env_id))
env = gym.make(task.env_id)
for trial in range(task.trials):
env = gym.make(task.env_id)
training_dir_name = "{}/{}-{}".format(args.training_dir, task.env_id, trial)
env.monitor.start(training_dir_name)
env = wrappers.Monitor(training_dir_name)(env)
env.reset()
for _ in range(task.max_timesteps):
o, r, done, _ = env.step(env.action_space.sample())
if done:
env.reset()
env.monitor.close()
env.close()
logger.info("""Done running, upload results using the following command:

69
examples/utilities/live_plot.py
View File

@@ -1,69 +0,0 @@
import gym
import matplotlib
import matplotlib.pyplot as plt
class LivePlot(object):
def __init__(self, outdir, data_key='episode_rewards', line_color='blue'):
"""
Liveplot renders a graph of either episode_rewards or episode_lengths
Args:
outdir (outdir): Monitor output file location used to populate the graph
data_key (Optional[str]): The key in the json to graph (episode_rewards or episode_lengths).
line_color (Optional[dict]): Color of the plot.
"""
self.outdir = outdir
self._last_data = None
self.data_key = data_key
self.line_color = line_color
#styling options
matplotlib.rcParams['toolbar'] = 'None'
plt.style.use('ggplot')
plt.xlabel("")
plt.ylabel(data_key)
fig = plt.gcf().canvas.set_window_title('')
def plot(self):
results = gym.monitoring.monitor.load_results(self.outdir)
data = results[self.data_key]
#only update plot if data is different (plot calls are expensive)
if data != self._last_data:
self._last_data = data
plt.plot(data, color=self.line_color)
# pause so matplotlib will display
# may want to figure out matplotlib animation or use a different library in the future
plt.pause(0.000001)
if __name__ == '__main__':
env = gym.make('CartPole-v0')
outdir = '/tmp/random-agent-results'
env.seed(0)
env.monitor.start(outdir, force=True)
# You may optionally include a LivePlot so that you can see
# how your agent is performing. Use plotter.plot() to update
# the graph.
plotter = LivePlot(outdir)
episode_count = 100
max_steps = 200
reward = 0
done = False
for i in range(episode_count):
ob = env.reset()
for j in range(max_steps):
ob, reward, done, _ = env.step(env.action_space.sample())
if done:
break
plotter.plot()
env.render()
# Dump result info to disk
env.monitor.close()

12
gym/benchmarks/tests/test_benchmark.py
View File

@@ -1,7 +1,7 @@
import numpy as np
import gym
from gym import monitoring
from gym import monitoring, wrappers
from gym.monitoring.tests import helpers
from gym.benchmarks import registration, scoring
@@ -22,20 +22,20 @@ def test():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env = wrappers.Monitor(directory=temp, video_callable=False)(env)
env.seed(0)
env.monitor.start(temp, video_callable=False)
env.monitor.configure(mode='evaluation')
env.set_monitor_mode('evaluation')
rollout(env)
env.monitor.configure(mode='training')
env.set_monitor_mode('training')
for i in range(2):
rollout(env)
env.monitor.configure(mode='evaluation')
env.set_monitor_mode('evaluation')
rollout(env, good=True)
env.monitor.close()
env.close()
results = monitoring.load_results(temp)
evaluation_score = benchmark.score_evaluation('CartPole-v0', results['data_sources'], results['initial_reset_timestamps'], results['episode_lengths'], results['episode_rewards'], results['episode_types'], results['timestamps'])
benchmark_score = benchmark.score_benchmark({

43
gym/core.py
View File

@@ -2,9 +2,8 @@ import logging
logger = logging.getLogger(__name__)
import numpy as np
import weakref
from gym import error, monitoring
from gym import error
from gym.utils import closer, reraise
env_closer = closer.Closer()
@@ -90,17 +89,7 @@ class Env(object):
@property
def monitor(self):
"""Lazily creates a monitor instance.
We do this lazily rather than at environment creation time
since when the monitor closes, we need remove the existing
monitor but also make it easy to start a new one. We could
still just forcibly create a new monitor instance on old
monitor close, but that seems less clean.
"""
if not hasattr(self, '_monitor'):
self._monitor = monitoring.Monitor(self)
return self._monitor
raise error.Error('env.monitor is deprecated. Wrap your env with gym.wrappers.Monitor to record data.')
def step(self, action):
"""Run one timestep of the environment's dynamics. When end of
@@ -118,10 +107,7 @@ class Env(object):
done (boolean): whether the episode has ended, in which case further step() calls will return undefined results
info (dict): contains auxiliary diagnostic information (helpful for debugging, and sometimes learning)
"""
self.monitor._before_step(action)
observation, reward, done, info = self._step(action)
done = self.monitor._after_step(observation, reward, done, info)
return observation, reward, done, info
def reset(self):
@@ -135,10 +121,7 @@ class Env(object):
raise error.Error("{} requires manually calling 'configure()' before 'reset()'".format(self))
elif not self._configured:
self.configure()
self.monitor._before_reset()
observation = self._reset()
self.monitor._after_reset(observation)
return observation
def render(self, mode='human', close=False):
@@ -202,9 +185,6 @@ class Env(object):
if not hasattr(self, '_closed') or self._closed:
return
# Automatically close the monitor and any render window.
if hasattr(self, '_monitor'):
self.monitor.close()
if self._owns_render:
self.render(close=True)
@@ -330,6 +310,25 @@ class Wrapper(Env):
self._spec = self.env.spec
self._unwrapped = self.env.unwrapped
self._update_wrapper_stack()
def _update_wrapper_stack(self):
"""
Keep a list of all the wrappers that have been appended to the stack.
"""
self._wrapper_stack = getattr(self.env, '_wrapper_stack', [])
self._check_for_duplicate_wrappers()
self._wrapper_stack.append(self)
def _check_for_duplicate_wrappers(self):
"""Raise an error if there are duplicate wrappers. Can be overwritten by subclasses"""
if self.class_name() in [wrapper.class_name() for wrapper in self._wrapper_stack]:
raise error.DoubleWrapperError("Attempted to double wrap with Wrapper: {}".format(self.class_name()))
@classmethod
def class_name(cls):
return cls.__name__
def _step(self, action):
return self.env.step(action)

5
gym/envs/safety/semisuper.py
View File

@@ -15,12 +15,11 @@ import gym
class SemisuperEnv(gym.Env):
def step(self, action):
assert self.action_space.contains(action)
self.monitor._before_step(action)
observation, true_reward, done, info = self._step(action)
assert self.observation_space.contains(observation)
info['true_reward'] = true_reward # Used by monitor for evaluating performance
done = self.monitor._after_step(observation, true_reward, done, info)
assert self.observation_space.contains(observation)
perceived_reward = self._distort_reward(true_reward)
return observation, perceived_reward, done, info

5
gym/envs/tests/test_envs.py
View File

@@ -27,6 +27,11 @@ def should_skip_env_spec_for_tests(spec):
logger.warn("Skipping tests for parameter_tuning env {}".format(spec._entry_point))
return True
# Skip Semisuper tests for now (broken due to monitor refactor)
if spec._entry_point.startswith('gym.envs.safety:Semisuper'):
logger.warn("Skipping tests for semisuper env {}".format(spec._entry_point))
return True
return False

12
gym/envs/tests/test_safety_envs.py Normal file
View File

@@ -0,0 +1,12 @@
import gym
def test_semisuper_true_rewards():
env = gym.make('SemisuperPendulumNoise-v0')
env.reset()
observation, perceived_reward, done, info = env.step(env.action_space.sample())
true_reward = info['true_reward']
# The noise in the reward should ensure these are different. If we get spurious errors, we can remove this check
assert perceived_reward != true_reward

5
gym/error.py
View File

@@ -125,3 +125,8 @@ class VideoRecorderError(Error):
class InvalidFrame(Error):
pass
# Wrapper errors
class DoubleWrapperError(Error):
pass

4
gym/monitoring/__init__.py
View File

@@ -1,9 +1,9 @@
from gym.monitoring.monitor import (
from gym.monitoring.monitor_manager import (
_open_monitors,
detect_training_manifests,
load_env_info_from_manifests,
load_results,
Monitor,
MonitorManager,
)
from gym.monitoring.stats_recorder import StatsRecorder
from gym.monitoring.video_recorder import VideoRecorder

48
gym/monitoring/monitor.py → gym/monitoring/monitor_manager.py
View File

@@ -1,16 +1,13 @@
import atexit
import logging
import json
import numpy as np
import logging
import os
import six
import sys
import threading
import weakref
import numpy as np
import six
from gym import error, version
from gym.monitoring import stats_recorder, video_recorder
from gym.utils import atomic_write, closer, seeding
from gym.utils import atomic_write, closer
logger = logging.getLogger(__name__)
@@ -50,7 +47,7 @@ monitor_closer = closer.Closer()
def _open_monitors():
return list(monitor_closer.closeables.values())
class Monitor(object):
class MonitorManager(object):
"""A configurable monitor for your training runs.
Every env has an attached monitor, which you can access as
@@ -67,7 +64,7 @@ class Monitor(object):
can also use 'monitor.configure(video_callable=lambda count: False)' to disable
video.
Monitor supports multiple threads and multiple processes writing
MonitorManager supports multiple threads and multiple processes writing
to the same directory of training data. The data will later be
joined by scoreboard.upload_training_data and on the server.
@@ -132,6 +129,7 @@ class Monitor(object):
video_callable = disable_videos
elif not callable(video_callable):
raise error.Error('You must provide a function, None, or False for video_callable, not {}: {}'.format(type(video_callable), video_callable))
self.video_callable = video_callable
# Check on whether we need to clear anything
if force:
@@ -143,7 +141,6 @@ class Monitor(object):
You should use a unique directory for each training run, or use 'force=True' to automatically clear previous monitor files.'''.format(directory, ', '.join(training_manifests[:5])))
self._monitor_id = monitor_closer.register(self)
self.enabled = True
@@ -154,7 +151,7 @@ class Monitor(object):
self.file_infix = '{}.{}'.format(self._monitor_id, uid if uid else os.getpid())
self.stats_recorder = stats_recorder.StatsRecorder(directory, '{}.episode_batch.{}'.format(self.file_prefix, self.file_infix), autoreset=self.env_semantics_autoreset, env_id=env_id)
self.configure(video_callable=video_callable)
if not os.path.exists(directory):
os.mkdir(directory)
self.write_upon_reset = write_upon_reset
@@ -162,7 +159,7 @@ class Monitor(object):
if mode is not None:
self._set_mode(mode)
def flush(self, force=False):
def _flush(self, force=False):
"""Flush all relevant monitor information to disk."""
if not self.write_upon_reset and not force:
return
@@ -192,7 +189,7 @@ class Monitor(object):
self.stats_recorder.close()
if self.video_recorder is not None:
self._close_video_recorder()
self.flush(force=True)
self._flush(force=True)
env = self._env_ref()
# Only take action if the env hasn't been GC'd
@@ -222,22 +219,6 @@ class Monitor(object):
logger.info('''Finished writing results. You can upload them to the scoreboard via gym.upload(%r)''', self.directory)
def configure(self, video_callable=None, mode=None):
"""Reconfigure the monitor.
video_callable (function): Whether to record video to upload to the scoreboard.
mode (['evaluation', 'training']): Whether this is an evaluation or training episode.
"""
if not self.enabled:
raise error.Error('Can only configure an enabled monitor. (HINT: did you already close this monitor?)')
if video_callable is not None:
self.video_callable = video_callable
if mode is not None:
self._set_mode(mode)
def _set_mode(self, mode):
if mode == 'evaluation':
type = 'e'
@@ -263,7 +244,10 @@ class Monitor(object):
# For envs with BlockingReset wrapping VNCEnv, this observation will be the first one of the new episode
self._reset_video_recorder()
self.episode_id += 1
self.flush()
self._flush()
if info.get('true_reward', None): # Semisupervised envs modify the rewards, but we want the original when scoring
reward = info['true_reward']
# Record stats
self.stats_recorder.after_step(observation, reward, done, info)
@@ -272,7 +256,6 @@ class Monitor(object):
return done
def _before_reset(self):
if not self.enabled: return
self.stats_recorder.before_reset()
@@ -288,10 +271,9 @@ class Monitor(object):
# Bump *after* all reset activity has finished
self.episode_id += 1
self.flush()
self._flush()
def _reset_video_recorder(self):
# Close any existing video recorder
if self.video_recorder:
self._close_video_recorder()

67
gym/monitoring/tests/test_monitor.py
View File

@@ -4,18 +4,15 @@ import os
import gym
from gym import error, spaces
from gym import monitoring
from gym.monitoring import monitor
from gym.monitoring.tests import helpers
from gym.wrappers import Monitor
class FakeEnv(gym.Env):
def _render(self, close=True):
raise RuntimeError('Raising')
def test_monitor_filename():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp)
env.monitor.close()
env = Monitor(directory=temp)(env)
env.close()
manifests = glob.glob(os.path.join(temp, '*.manifest.*'))
assert len(manifests) == 1
@@ -23,43 +20,34 @@ def test_monitor_filename():
def test_write_upon_reset_false():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp, video_callable=False, write_upon_reset=False)
env = Monitor(directory=temp, video_callable=False, write_upon_reset=False)(env)
env.reset()
files = glob.glob(os.path.join(temp, '*'))
assert not files, "Files: {}".format(files)
env.monitor.close()
env.close()
files = glob.glob(os.path.join(temp, '*'))
assert len(files) > 0
def test_write_upon_reset_true():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp, video_callable=False, write_upon_reset=True)
env = Monitor(directory=temp, video_callable=False, write_upon_reset=True)(env)
env.reset()
files = glob.glob(os.path.join(temp, '*'))
assert len(files) > 0, "Files: {}".format(files)
env.monitor.close()
env.close()
files = glob.glob(os.path.join(temp, '*'))
assert len(files) > 0
def test_close_monitor():
with helpers.tempdir() as temp:
env = FakeEnv()
env.monitor.start(temp)
env.monitor.close()
manifests = monitor.detect_training_manifests(temp)
assert len(manifests) == 1
def test_video_callable_true_not_allowed():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
try:
env.monitor.start(temp, video_callable=True)
env = Monitor(temp, video_callable=True)(env)
except error.Error:
pass
else:
@@ -68,35 +56,29 @@ def test_video_callable_true_not_allowed():
def test_video_callable_false_does_not_record():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp, video_callable=False)
env = Monitor(temp, video_callable=False)(env)
env.reset()
env.monitor.close()
env.close()
results = monitoring.load_results(temp)
assert len(results['videos']) == 0
def test_video_callable_records_videos():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp)
env = Monitor(temp)(env)
env.reset()
env.monitor.close()
env.close()
results = monitoring.load_results(temp)
assert len(results['videos']) == 1, "Videos: {}".format(results['videos'])
def test_env_reuse():
def test_semisuper_succeeds():
"""Regression test. Ensure that this can write"""
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp)
env.monitor.close()
env.monitor.start(temp, force=True)
env = gym.make('SemisuperPendulumDecay-v0')
env = Monitor(temp)(env)
env.reset()
env.step(env.action_space.sample())
env.step(env.action_space.sample())
env.monitor.close()
results = monitor.load_results(temp)
assert results['episode_lengths'] == [2], 'Results: {}'.format(results)
env.close()
class AutoresetEnv(gym.Env):
metadata = {'semantics.autoreset': True}
@@ -111,6 +93,8 @@ class AutoresetEnv(gym.Env):
def _step(self, action):
return 0, 0, False, {}
import logging
logger = logging.getLogger()
gym.envs.register(
id='Autoreset-v0',
entry_point='gym.monitoring.tests.test_monitor:AutoresetEnv',
@@ -119,7 +103,7 @@ gym.envs.register(
def test_env_reuse():
with helpers.tempdir() as temp:
env = gym.make('Autoreset-v0')
env.monitor.start(temp)
env = Monitor(temp)(env)
env.reset()
@@ -131,6 +115,8 @@ def test_env_reuse():
_, _, done, _ = env.step(None)
assert done
env.close()
def test_no_monitor_reset_unless_done():
def assert_reset_raises(env):
errored = False
@@ -149,7 +135,7 @@ def test_no_monitor_reset_unless_done():
env.reset()
# can reset once as soon as we start
env.monitor.start(temp, video_callable=False)
env = Monitor(temp, video_callable=False)(env)
env.reset()
# can reset multiple times in a row
@@ -171,13 +157,12 @@ def test_no_monitor_reset_unless_done():
env.step(env.action_space.sample())
assert_reset_raises(env)
env.monitor.close()
env.close()
def test_only_complete_episodes_written():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env.monitor.start(temp, video_callable=False)
env = Monitor(temp, video_callable=False)(env)
env.reset()
d = False
while not d:
@@ -186,7 +171,7 @@ def test_only_complete_episodes_written():
env.reset()
env.step(env.action_space.sample())
env.monitor.close()
env.close()
# Only 1 episode should be written
results = monitoring.load_results(temp)

41
gym/monitoring/tests/test_monitor_envs.py
View File

@@ -1,41 +0,0 @@
import numpy as np
from nose2 import tools
import os
import logging
logger = logging.getLogger(__name__)
from gym import envs
from gym.monitoring.tests import helpers
specs = [spec for spec in sorted(envs.registry.all(), key=lambda x: x.id) if spec._entry_point is not None]
@tools.params(*specs)
def test_renderable_after_monitor_close(spec):
# TODO(gdb 2016-05-15): Re-enable these tests after fixing box2d-py
if spec._entry_point.startswith('gym.envs.box2d:'):
logger.warn("Skipping tests for box2d env {}".format(spec._entry_point))
return
elif spec._entry_point.startswith('gym.envs.parameter_tuning:'):
logger.warn("Skipping tests for parameter tuning".format(spec._entry_point))
return
# Skip mujoco tests
skip_mujoco = not (os.environ.get('MUJOCO_KEY_BUNDLE') or os.path.exists(os.path.expanduser('~/.mujoco')))
if skip_mujoco and spec._entry_point.startswith('gym.envs.mujoco:'):
return
with helpers.tempdir() as temp:
env = spec.make()
# Skip un-renderable envs
if 'human' not in env.metadata.get('render.modes', []):
return
env.monitor.start(temp)
env.reset()
env.monitor.close()
env.reset()
env.render()
env.render(close=True)
env.close()

2
gym/scoreboard/api.py
View File

@@ -96,7 +96,7 @@ def _upload(training_dir, algorithm_id=None, writeup=None, benchmark_run_id=None
open_monitors = monitoring._open_monitors()
if len(open_monitors) > 0:
envs = [m.env.spec.id if m.env.spec else '(unknown)' for m in open_monitors]
raise error.Error("Still have an open monitor on {}. You must run 'env.monitor.close()' before uploading.".format(', '.join(envs)))
raise error.Error("Still have an open monitor on {}. You must run 'env.close()' before uploading.".format(', '.join(envs)))
env_info, training_episode_batch, training_video = upload_training_data(training_dir, api_key=api_key)
env_id = env_info['env_id']

3
gym/wrappers/__init__.py
View File

@@ -1 +1,4 @@
from gym import error
from gym.wrappers.frame_skipping import SkipWrapper
from gym.wrappers.monitoring import Monitor

41
gym/wrappers/monitoring.py Normal file
View File

@@ -0,0 +1,41 @@
from gym import monitoring
from gym import Wrapper
import logging
logger = logging.getLogger(__name__)
def Monitor(directory, video_callable=None, force=False, resume=False,
write_upon_reset=False, uid=None, mode=None):
class Monitor(Wrapper):
def __init__(self, env):
super(Monitor, self).__init__(env)
self._monitor = monitoring.MonitorManager(env)
self._monitor.start(directory, video_callable, force, resume,
write_upon_reset, uid, mode)
def _step(self, action):
self._monitor._before_step(action)
observation, reward, done, info = self.env.step(action)
done = self._monitor._after_step(observation, reward, done, info)
return observation, reward, done, info
def _reset(self):
self._monitor._before_reset()
observation = self.env.reset()
self._monitor._after_reset(observation)
return observation
def _close(self):
super(Monitor, self)._close()
# _monitor will not be set if super(Monitor, self).__init__ raises, this check prevents a confusing error message
if getattr(self, '_monitor', None):
self._monitor.close()
def set_monitor_mode(self, mode):
logger.info("Setting the monitor mode is deprecated and will be removed soon")
self._monitor._set_mode(mode)
return Monitor

0
gym/wrappers/tests/__init__.py Normal file
View File

26
gym/wrappers/tests/test_wrappers.py
View File

@@ -1,9 +1,35 @@
import gym
from gym import error
from gym import wrappers
from gym.wrappers import SkipWrapper
import tempfile
import shutil
def test_skip():
every_two_frame = SkipWrapper(2)
env = gym.make("FrozenLake-v0")
env = every_two_frame(env)
obs = env.reset()
env.render()
def test_no_double_wrapping():
temp = tempfile.mkdtemp()
try:
env = gym.make("FrozenLake-v0")
env = wrappers.Monitor(temp)(env)
try:
env = wrappers.Monitor(temp)(env)
except error.DoubleWrapperError:
pass
else:
assert False, "Should not allow double wrapping"
env.close()
finally:
shutil.rmtree(temp)
if __name__ == '__main__':
test_no_double_wrapping()