mirror of
https://github.com/Farama-Foundation/Gymnasium.git
synced 2025-07-31 22:04:31 +00:00
e913bc81b8
* feat: add `isort` to `pre-commit` * ci: skip `__init__.py` file for `isort` * ci: make `isort` mandatory in lint pipeline * docs: add a section on Git hooks * ci: check isort diff * fix: isort from master branch * docs: add pre-commit badge * ci: update black + bandit versions * feat: add PR template * refactor: PR template * ci: remove bandit * docs: add Black badge * ci: try to remove all `|| true` statements * ci: remove lint_python job - Remove `lint_python` CI job - Move `pyupgrade` job to `pre-commit` workflow * fix: avoid messing with typing * docs: add a note on running `pre-cpmmit` manually * ci: apply `pre-commit` to the whole codebase
116 lines
3.6 KiB
Python
116 lines
3.6 KiB
Python
from typing import Optional
|
|
|
|
import numpy as np
|
|
|
|
import gym
|
|
|
|
|
|
# taken from https://github.com/openai/baselines/blob/master/baselines/common/vec_env/vec_normalize.py
|
|
class RunningMeanStd:
|
|
# https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm
|
|
def __init__(self, epsilon=1e-4, shape=()):
|
|
self.mean = np.zeros(shape, "float64")
|
|
self.var = np.ones(shape, "float64")
|
|
self.count = epsilon
|
|
|
|
def update(self, x):
|
|
batch_mean = np.mean(x, axis=0)
|
|
batch_var = np.var(x, axis=0)
|
|
batch_count = x.shape[0]
|
|
self.update_from_moments(batch_mean, batch_var, batch_count)
|
|
|
|
def update_from_moments(self, batch_mean, batch_var, batch_count):
|
|
self.mean, self.var, self.count = update_mean_var_count_from_moments(
|
|
self.mean, self.var, self.count, batch_mean, batch_var, batch_count
|
|
)
|
|
|
|
|
|
def update_mean_var_count_from_moments(
|
|
mean, var, count, batch_mean, batch_var, batch_count
|
|
):
|
|
delta = batch_mean - mean
|
|
tot_count = count + batch_count
|
|
|
|
new_mean = mean + delta * batch_count / tot_count
|
|
m_a = var * count
|
|
m_b = batch_var * batch_count
|
|
M2 = m_a + m_b + np.square(delta) * count * batch_count / tot_count
|
|
new_var = M2 / tot_count
|
|
new_count = tot_count
|
|
|
|
return new_mean, new_var, new_count
|
|
|
|
|
|
class NormalizeObservation(gym.core.Wrapper):
|
|
def __init__(
|
|
self,
|
|
env,
|
|
epsilon=1e-8,
|
|
):
|
|
super().__init__(env)
|
|
self.num_envs = getattr(env, "num_envs", 1)
|
|
self.is_vector_env = getattr(env, "is_vector_env", False)
|
|
if self.is_vector_env:
|
|
self.obs_rms = RunningMeanStd(shape=self.single_observation_space.shape)
|
|
else:
|
|
self.obs_rms = RunningMeanStd(shape=self.observation_space.shape)
|
|
self.epsilon = epsilon
|
|
|
|
def step(self, action):
|
|
obs, rews, dones, infos = self.env.step(action)
|
|
if self.is_vector_env:
|
|
obs = self.normalize(obs)
|
|
else:
|
|
obs = self.normalize(np.array([obs]))[0]
|
|
return obs, rews, dones, infos
|
|
|
|
def reset(self, **kwargs):
|
|
return_info = kwargs.get("return_info", False)
|
|
if return_info:
|
|
obs, info = self.env.reset(**kwargs)
|
|
else:
|
|
obs = self.env.reset(**kwargs)
|
|
if self.is_vector_env:
|
|
obs = self.normalize(obs)
|
|
else:
|
|
obs = self.normalize(np.array([obs]))[0]
|
|
if not return_info:
|
|
return obs
|
|
else:
|
|
return obs, info
|
|
|
|
def normalize(self, obs):
|
|
self.obs_rms.update(obs)
|
|
return (obs - self.obs_rms.mean) / np.sqrt(self.obs_rms.var + self.epsilon)
|
|
|
|
|
|
class NormalizeReward(gym.core.Wrapper):
|
|
def __init__(
|
|
self,
|
|
env,
|
|
gamma=0.99,
|
|
epsilon=1e-8,
|
|
):
|
|
super().__init__(env)
|
|
self.num_envs = getattr(env, "num_envs", 1)
|
|
self.is_vector_env = getattr(env, "is_vector_env", False)
|
|
self.return_rms = RunningMeanStd(shape=())
|
|
self.returns = np.zeros(self.num_envs)
|
|
self.gamma = gamma
|
|
self.epsilon = epsilon
|
|
|
|
def step(self, action):
|
|
obs, rews, dones, infos = self.env.step(action)
|
|
if not self.is_vector_env:
|
|
rews = np.array([rews])
|
|
self.returns = self.returns * self.gamma + rews
|
|
rews = self.normalize(rews)
|
|
self.returns[dones] = 0.0
|
|
if not self.is_vector_env:
|
|
rews = rews[0]
|
|
return obs, rews, dones, infos
|
|
|
|
def normalize(self, rews):
|
|
self.return_rms.update(self.returns)
|
|
return rews / np.sqrt(self.return_rms.var + self.epsilon)
|