__credits__ = ["Andrea PIERRÉ"] import math import warnings from typing import Optional import numpy as np import gym from gym import error, spaces from gym.error import DependencyNotInstalled from gym.utils import EzPickle, colorize from gym.utils.renderer import Renderer try: import Box2D from Box2D.b2 import ( circleShape, contactListener, edgeShape, fixtureDef, polygonShape, revoluteJointDef, ) except ImportError: raise DependencyNotInstalled("box2d is not installed, run `pip install gym[box2d]`") FPS = 50 SCALE = 30.0 # affects how fast-paced the game is, forces should be adjusted as well MAIN_ENGINE_POWER = 13.0 SIDE_ENGINE_POWER = 0.6 INITIAL_RANDOM = 1000.0 # Set 1500 to make game harder LANDER_POLY = [(-14, +17), (-17, 0), (-17, -10), (+17, -10), (+17, 0), (+14, +17)] LEG_AWAY = 20 LEG_DOWN = 18 LEG_W, LEG_H = 2, 8 LEG_SPRING_TORQUE = 40 SIDE_ENGINE_HEIGHT = 14.0 SIDE_ENGINE_AWAY = 12.0 VIEWPORT_W = 600 VIEWPORT_H = 400 class ContactDetector(contactListener): def __init__(self, env): contactListener.__init__(self) self.env = env def BeginContact(self, contact): if ( self.env.lander == contact.fixtureA.body or self.env.lander == contact.fixtureB.body ): self.env.game_over = True for i in range(2): if self.env.legs[i] in [contact.fixtureA.body, contact.fixtureB.body]: self.env.legs[i].ground_contact = True def EndContact(self, contact): for i in range(2): if self.env.legs[i] in [contact.fixtureA.body, contact.fixtureB.body]: self.env.legs[i].ground_contact = False class LunarLander(gym.Env, EzPickle): """ ### Description This environment is a classic rocket trajectory optimization problem. According to Pontryagin's maximum principle, it is optimal to fire the engine at full throttle or turn it off. This is the reason why this environment has discrete actions: engine on or off. There are two environment versions: discrete or continuous. The landing pad is always at coordinates (0,0). The coordinates are the first two numbers in the state vector. Landing outside of the landing pad is possible. Fuel is infinite, so an agent can learn to fly and then land on its first attempt. To see a heuristic landing, run: ``` python gym/envs/box2d/lunar_lander.py ``` ### Action Space There are four discrete actions available: do nothing, fire left orientation engine, fire main engine, fire right orientation engine. ### Observation Space There are 8 states: the coordinates of the lander in `x` & `y`, its linear velocities in `x` & `y`, its angle, its angular velocity, and two booleans that represent whether each leg is in contact with the ground or not. ### Rewards Reward for moving from the top of the screen to the landing pad and coming to rest is about 100-140 points. If the lander moves away from the landing pad, it loses reward. If the lander crashes, it receives an additional -100 points. If it comes to rest, it receives an additional +100 points. Each leg with ground contact is +10 points. Firing the main engine is -0.3 points each frame. Firing the side engine is -0.03 points each frame. Solved is 200 points. ### Starting State The lander starts at the top center of the viewport with a random initial force applied to its center of mass. ### Episode Termination The episode finishes if: 1) the lander crashes (the lander body gets in contact with the moon); 2) the lander gets outside of the viewport (`x` coordinate is greater than 1); 3) the lander is not awake. From the [Box2D docs](https://box2d.org/documentation/md__d_1__git_hub_box2d_docs_dynamics.html#autotoc_md61), a body which is not awake is a body which doesn't move and doesn't collide with any other body: > When Box2D determines that a body (or group of bodies) has come to rest, > the body enters a sleep state which has very little CPU overhead. If a > body is awake and collides with a sleeping body, then the sleeping body > wakes up. Bodies will also wake up if a joint or contact attached to > them is destroyed. ### Arguments To use to the _continuous_ environment, you need to specify the `continuous=True` argument like below: ```python import gym env = gym.make( "LunarLander-v2", continuous: bool = False, gravity: float = -10.0, enable_wind: bool = False, wind_power: float = 15.0, turbulence_power: float = 1.5, ) ``` If `continuous=True` is passed, continuous actions (corresponding to the throttle of the engines) will be used and the action space will be `Box(-1, +1, (2,), dtype=np.float32)`. The first coordinate of an action determines the throttle of the main engine, while the second coordinate specifies the throttle of the lateral boosters. Given an action `np.array([main, lateral])`, the main engine will be turned off completely if `main < 0` and the throttle scales affinely from 50% to 100% for `0 <= main <= 1` (in particular, the main engine doesn't work with less than 50% power). Similarly, if `-0.5 < lateral < 0.5`, the lateral boosters will not fire at all. If `lateral < -0.5`, the left booster will fire, and if `lateral > 0.5`, the right booster will fire. Again, the throttle scales affinely from 50% to 100% between -1 and -0.5 (and 0.5 and 1, respectively). `gravity` dictates the gravitational constant, this is bounded to be within 0 and -12. If `enable_wind=True` is passed, there will be wind effects applied to the lander. The wind is generated using the function `tanh(sin(2 k (t+C)) + sin(pi k (t+C)))`. `k` is set to 0.01. `C` is sampled randomly between -9999 and 9999. `wind_power` dictates the maximum magnitude of linear wind applied to the craft. The recommended value for `wind_power` is between 0.0 and 20.0. `turbulence_power` dictates the maximum magnitude of rotational wind applied to the craft. The recommended value for `turbulence_power` is between 0.0 and 2.0. ### Version History - v2: Count energy spent and in v0.24, added turbulance with wind power and turbulence_power parameters - v1: Legs contact with ground added in state vector; contact with ground give +10 reward points, and -10 if then lose contact; reward renormalized to 200; harder initial random push. - v0: Initial version ### Credits Created by Oleg Klimov """ metadata = { "render_modes": ["human", "rgb_array", "single_rgb_array"], "render_fps": FPS, } def __init__( self, render_mode: Optional[str] = None, continuous: bool = False, gravity: float = -10.0, enable_wind: bool = False, wind_power: float = 15.0, turbulence_power: float = 1.5, ): EzPickle.__init__(self) assert ( -12.0 < gravity and gravity < 0.0 ), f"gravity (current value: {gravity}) must be between -12 and 0" self.gravity = gravity if 0.0 > wind_power or wind_power > 20.0: warnings.warn( colorize( f"WARN: wind_power value is recommended to be between 0.0 and 20.0, (current value: {wind_power})", "yellow", ), ) self.wind_power = wind_power if 0.0 > turbulence_power or turbulence_power > 2.0: warnings.warn( colorize( f"WARN: turbulence_power value is recommended to be between 0.0 and 2.0, (current value: {turbulence_power})", "yellow", ), ) self.turbulence_power = turbulence_power self.enable_wind = enable_wind self.wind_idx = np.random.randint(-9999, 9999) self.torque_idx = np.random.randint(-9999, 9999) self.screen = None self.clock = None self.isopen = True self.world = Box2D.b2World(gravity=(0, gravity)) self.moon = None self.lander = None self.particles = [] self.prev_reward = None self.continuous = continuous low = np.array( [ # these are bounds for position # realistically the environment should have ended # long before we reach more than 50% outside -1.5, -1.5, # velocity bounds is 5x rated speed -5.0, -5.0, -math.pi, -5.0, -0.0, -0.0, ] ).astype(np.float32) high = np.array( [ # these are bounds for position # realistically the environment should have ended # long before we reach more than 50% outside 1.5, 1.5, # velocity bounds is 5x rated speed 5.0, 5.0, math.pi, 5.0, 1.0, 1.0, ] ).astype(np.float32) # useful range is -1 .. +1, but spikes can be higher self.observation_space = spaces.Box(low, high) if self.continuous: # Action is two floats [main engine, left-right engines]. # Main engine: -1..0 off, 0..+1 throttle from 50% to 100% power. Engine can't work with less than 50% power. # Left-right: -1.0..-0.5 fire left engine, +0.5..+1.0 fire right engine, -0.5..0.5 off self.action_space = spaces.Box(-1, +1, (2,), dtype=np.float32) else: # Nop, fire left engine, main engine, right engine self.action_space = spaces.Discrete(4) self.render_mode = render_mode self.renderer = Renderer(self.render_mode, self._render) def _destroy(self): if not self.moon: return self.world.contactListener = None self._clean_particles(True) self.world.DestroyBody(self.moon) self.moon = None self.world.DestroyBody(self.lander) self.lander = None self.world.DestroyBody(self.legs[0]) self.world.DestroyBody(self.legs[1]) def reset( self, *, seed: Optional[int] = None, return_info: bool = False, options: Optional[dict] = None, ): super().reset(seed=seed) self._destroy() self.world.contactListener_keepref = ContactDetector(self) self.world.contactListener = self.world.contactListener_keepref self.game_over = False self.prev_shaping = None W = VIEWPORT_W / SCALE H = VIEWPORT_H / SCALE # terrain CHUNKS = 11 height = self.np_random.uniform(0, H / 2, size=(CHUNKS + 1,)) chunk_x = [W / (CHUNKS - 1) * i for i in range(CHUNKS)] self.helipad_x1 = chunk_x[CHUNKS // 2 - 1] self.helipad_x2 = chunk_x[CHUNKS // 2 + 1] self.helipad_y = H / 4 height[CHUNKS // 2 - 2] = self.helipad_y height[CHUNKS // 2 - 1] = self.helipad_y height[CHUNKS // 2 + 0] = self.helipad_y height[CHUNKS // 2 + 1] = self.helipad_y height[CHUNKS // 2 + 2] = self.helipad_y smooth_y = [ 0.33 * (height[i - 1] + height[i + 0] + height[i + 1]) for i in range(CHUNKS) ] self.moon = self.world.CreateStaticBody( shapes=edgeShape(vertices=[(0, 0), (W, 0)]) ) self.sky_polys = [] for i in range(CHUNKS - 1): p1 = (chunk_x[i], smooth_y[i]) p2 = (chunk_x[i + 1], smooth_y[i + 1]) self.moon.CreateEdgeFixture(vertices=[p1, p2], density=0, friction=0.1) self.sky_polys.append([p1, p2, (p2[0], H), (p1[0], H)]) self.moon.color1 = (0.0, 0.0, 0.0) self.moon.color2 = (0.0, 0.0, 0.0) initial_y = VIEWPORT_H / SCALE self.lander = self.world.CreateDynamicBody( position=(VIEWPORT_W / SCALE / 2, initial_y), angle=0.0, fixtures=fixtureDef( shape=polygonShape( vertices=[(x / SCALE, y / SCALE) for x, y in LANDER_POLY] ), density=5.0, friction=0.1, categoryBits=0x0010, maskBits=0x001, # collide only with ground restitution=0.0, ), # 0.99 bouncy ) self.lander.color1 = (128, 102, 230) self.lander.color2 = (77, 77, 128) self.lander.ApplyForceToCenter( ( self.np_random.uniform(-INITIAL_RANDOM, INITIAL_RANDOM), self.np_random.uniform(-INITIAL_RANDOM, INITIAL_RANDOM), ), True, ) self.legs = [] for i in [-1, +1]: leg = self.world.CreateDynamicBody( position=(VIEWPORT_W / SCALE / 2 - i * LEG_AWAY / SCALE, initial_y), angle=(i * 0.05), fixtures=fixtureDef( shape=polygonShape(box=(LEG_W / SCALE, LEG_H / SCALE)), density=1.0, restitution=0.0, categoryBits=0x0020, maskBits=0x001, ), ) leg.ground_contact = False leg.color1 = (128, 102, 230) leg.color2 = (77, 77, 128) rjd = revoluteJointDef( bodyA=self.lander, bodyB=leg, localAnchorA=(0, 0), localAnchorB=(i * LEG_AWAY / SCALE, LEG_DOWN / SCALE), enableMotor=True, enableLimit=True, maxMotorTorque=LEG_SPRING_TORQUE, motorSpeed=+0.3 * i, # low enough not to jump back into the sky ) if i == -1: rjd.lowerAngle = ( +0.9 - 0.5 ) # The most esoteric numbers here, angled legs have freedom to travel within rjd.upperAngle = +0.9 else: rjd.lowerAngle = -0.9 rjd.upperAngle = -0.9 + 0.5 leg.joint = self.world.CreateJoint(rjd) self.legs.append(leg) self.drawlist = [self.lander] + self.legs self.renderer.reset() if not return_info: return self.step(np.array([0, 0]) if self.continuous else 0)[0] else: return self.step(np.array([0, 0]) if self.continuous else 0)[0], {} def _create_particle(self, mass, x, y, ttl): p = self.world.CreateDynamicBody( position=(x, y), angle=0.0, fixtures=fixtureDef( shape=circleShape(radius=2 / SCALE, pos=(0, 0)), density=mass, friction=0.1, categoryBits=0x0100, maskBits=0x001, # collide only with ground restitution=0.3, ), ) p.ttl = ttl self.particles.append(p) self._clean_particles(False) return p def _clean_particles(self, all): while self.particles and (all or self.particles[0].ttl < 0): self.world.DestroyBody(self.particles.pop(0)) def step(self, action): # Update wind if self.enable_wind and not ( self.legs[0].ground_contact or self.legs[1].ground_contact ): # the function used for wind is tanh(sin(2 k x) + sin(pi k x)), # which is proven to never be periodic, k = 0.01 wind_mag = ( math.tanh( math.sin(0.02 * self.wind_idx) + (math.sin(math.pi * 0.01 * self.wind_idx)) ) * self.wind_power ) self.wind_idx += 1 self.lander.ApplyForceToCenter( (wind_mag, 0.0), True, ) # the function used for torque is tanh(sin(2 k x) + sin(pi k x)), # which is proven to never be periodic, k = 0.01 torque_mag = math.tanh( math.sin(0.02 * self.torque_idx) + (math.sin(math.pi * 0.01 * self.torque_idx)) ) * (self.turbulence_power) self.torque_idx += 1 self.lander.ApplyTorque( (torque_mag), True, ) if self.continuous: action = np.clip(action, -1, +1).astype(np.float32) else: assert self.action_space.contains( action ), f"{action!r} ({type(action)}) invalid " # Engines tip = (math.sin(self.lander.angle), math.cos(self.lander.angle)) side = (-tip[1], tip[0]) dispersion = [self.np_random.uniform(-1.0, +1.0) / SCALE for _ in range(2)] m_power = 0.0 if (self.continuous and action[0] > 0.0) or ( not self.continuous and action == 2 ): # Main engine if self.continuous: m_power = (np.clip(action[0], 0.0, 1.0) + 1.0) * 0.5 # 0.5..1.0 assert m_power >= 0.5 and m_power <= 1.0 else: m_power = 1.0 # 4 is move a bit downwards, +-2 for randomness ox = tip[0] * (4 / SCALE + 2 * dispersion[0]) + side[0] * dispersion[1] oy = -tip[1] * (4 / SCALE + 2 * dispersion[0]) - side[1] * dispersion[1] impulse_pos = (self.lander.position[0] + ox, self.lander.position[1] + oy) p = self._create_particle( 3.5, # 3.5 is here to make particle speed adequate impulse_pos[0], impulse_pos[1], m_power, ) # particles are just a decoration p.ApplyLinearImpulse( (ox * MAIN_ENGINE_POWER * m_power, oy * MAIN_ENGINE_POWER * m_power), impulse_pos, True, ) self.lander.ApplyLinearImpulse( (-ox * MAIN_ENGINE_POWER * m_power, -oy * MAIN_ENGINE_POWER * m_power), impulse_pos, True, ) s_power = 0.0 if (self.continuous and np.abs(action[1]) > 0.5) or ( not self.continuous and action in [1, 3] ): # Orientation engines if self.continuous: direction = np.sign(action[1]) s_power = np.clip(np.abs(action[1]), 0.5, 1.0) assert s_power >= 0.5 and s_power <= 1.0 else: direction = action - 2 s_power = 1.0 ox = tip[0] * dispersion[0] + side[0] * ( 3 * dispersion[1] + direction * SIDE_ENGINE_AWAY / SCALE ) oy = -tip[1] * dispersion[0] - side[1] * ( 3 * dispersion[1] + direction * SIDE_ENGINE_AWAY / SCALE ) impulse_pos = ( self.lander.position[0] + ox - tip[0] * 17 / SCALE, self.lander.position[1] + oy + tip[1] * SIDE_ENGINE_HEIGHT / SCALE, ) p = self._create_particle(0.7, impulse_pos[0], impulse_pos[1], s_power) p.ApplyLinearImpulse( (ox * SIDE_ENGINE_POWER * s_power, oy * SIDE_ENGINE_POWER * s_power), impulse_pos, True, ) self.lander.ApplyLinearImpulse( (-ox * SIDE_ENGINE_POWER * s_power, -oy * SIDE_ENGINE_POWER * s_power), impulse_pos, True, ) self.world.Step(1.0 / FPS, 6 * 30, 2 * 30) pos = self.lander.position vel = self.lander.linearVelocity state = [ (pos.x - VIEWPORT_W / SCALE / 2) / (VIEWPORT_W / SCALE / 2), (pos.y - (self.helipad_y + LEG_DOWN / SCALE)) / (VIEWPORT_H / SCALE / 2), vel.x * (VIEWPORT_W / SCALE / 2) / FPS, vel.y * (VIEWPORT_H / SCALE / 2) / FPS, self.lander.angle, 20.0 * self.lander.angularVelocity / FPS, 1.0 if self.legs[0].ground_contact else 0.0, 1.0 if self.legs[1].ground_contact else 0.0, ] assert len(state) == 8 reward = 0 shaping = ( -100 * np.sqrt(state[0] * state[0] + state[1] * state[1]) - 100 * np.sqrt(state[2] * state[2] + state[3] * state[3]) - 100 * abs(state[4]) + 10 * state[6] + 10 * state[7] ) # And ten points for legs contact, the idea is if you # lose contact again after landing, you get negative reward if self.prev_shaping is not None: reward = shaping - self.prev_shaping self.prev_shaping = shaping reward -= ( m_power * 0.30 ) # less fuel spent is better, about -30 for heuristic landing reward -= s_power * 0.03 done = False if self.game_over or abs(state[0]) >= 1.0: done = True reward = -100 if not self.lander.awake: done = True reward = +100 self.renderer.render_step() return np.array(state, dtype=np.float32), reward, done, {} def render(self, mode="human"): if self.render_mode is not None: return self.renderer.get_renders() else: return self._render(mode) def _render(self, mode="human"): assert mode in self.metadata["render_modes"] try: import pygame from pygame import gfxdraw except ImportError: raise DependencyNotInstalled( "pygame is not installed, run `pip install gym[box2d]`" ) if self.screen is None and mode == "human": pygame.init() pygame.display.init() self.screen = pygame.display.set_mode((VIEWPORT_W, VIEWPORT_H)) if self.clock is None: self.clock = pygame.time.Clock() self.surf = pygame.Surface((VIEWPORT_W, VIEWPORT_H)) pygame.transform.scale(self.surf, (SCALE, SCALE)) pygame.draw.rect(self.surf, (255, 255, 255), self.surf.get_rect()) for obj in self.particles: obj.ttl -= 0.15 obj.color1 = ( int(max(0.2, 0.15 + obj.ttl) * 255), int(max(0.2, 0.5 * obj.ttl) * 255), int(max(0.2, 0.5 * obj.ttl) * 255), ) obj.color2 = ( int(max(0.2, 0.15 + obj.ttl) * 255), int(max(0.2, 0.5 * obj.ttl) * 255), int(max(0.2, 0.5 * obj.ttl) * 255), ) self._clean_particles(False) for p in self.sky_polys: scaled_poly = [] for coord in p: scaled_poly.append((coord[0] * SCALE, coord[1] * SCALE)) pygame.draw.polygon(self.surf, (0, 0, 0), scaled_poly) gfxdraw.aapolygon(self.surf, scaled_poly, (0, 0, 0)) for obj in self.particles + self.drawlist: for f in obj.fixtures: trans = f.body.transform if type(f.shape) is circleShape: pygame.draw.circle( self.surf, color=obj.color1, center=trans * f.shape.pos * SCALE, radius=f.shape.radius * SCALE, ) pygame.draw.circle( self.surf, color=obj.color2, center=trans * f.shape.pos * SCALE, radius=f.shape.radius * SCALE, ) else: path = [trans * v * SCALE for v in f.shape.vertices] pygame.draw.polygon(self.surf, color=obj.color1, points=path) gfxdraw.aapolygon(self.surf, path, obj.color1) pygame.draw.aalines( self.surf, color=obj.color2, points=path, closed=True ) for x in [self.helipad_x1, self.helipad_x2]: x = x * SCALE flagy1 = self.helipad_y * SCALE flagy2 = flagy1 + 50 pygame.draw.line( self.surf, color=(255, 255, 255), start_pos=(x, flagy1), end_pos=(x, flagy2), width=1, ) pygame.draw.polygon( self.surf, color=(204, 204, 0), points=[ (x, flagy2), (x, flagy2 - 10), (x + 25, flagy2 - 5), ], ) gfxdraw.aapolygon( self.surf, [(x, flagy2), (x, flagy2 - 10), (x + 25, flagy2 - 5)], (204, 204, 0), ) self.surf = pygame.transform.flip(self.surf, False, True) if mode == "human": self.screen.blit(self.surf, (0, 0)) pygame.event.pump() self.clock.tick(self.metadata["render_fps"]) pygame.display.flip() elif mode in {"rgb_array", "single_rgb_array"}: return np.transpose( np.array(pygame.surfarray.pixels3d(self.surf)), axes=(1, 0, 2) ) def close(self): if self.screen is not None: import pygame pygame.display.quit() pygame.quit() self.isopen = False def heuristic(env, s): """ The heuristic for 1. Testing 2. Demonstration rollout. Args: env: The environment s (list): The state. Attributes: s[0] is the horizontal coordinate s[1] is the vertical coordinate s[2] is the horizontal speed s[3] is the vertical speed s[4] is the angle s[5] is the angular speed s[6] 1 if first leg has contact, else 0 s[7] 1 if second leg has contact, else 0 Returns: a: The heuristic to be fed into the step function defined above to determine the next step and reward. """ angle_targ = s[0] * 0.5 + s[2] * 1.0 # angle should point towards center if angle_targ > 0.4: angle_targ = 0.4 # more than 0.4 radians (22 degrees) is bad if angle_targ < -0.4: angle_targ = -0.4 hover_targ = 0.55 * np.abs( s[0] ) # target y should be proportional to horizontal offset angle_todo = (angle_targ - s[4]) * 0.5 - (s[5]) * 1.0 hover_todo = (hover_targ - s[1]) * 0.5 - (s[3]) * 0.5 if s[6] or s[7]: # legs have contact angle_todo = 0 hover_todo = ( -(s[3]) * 0.5 ) # override to reduce fall speed, that's all we need after contact if env.continuous: a = np.array([hover_todo * 20 - 1, -angle_todo * 20]) a = np.clip(a, -1, +1) else: a = 0 if hover_todo > np.abs(angle_todo) and hover_todo > 0.05: a = 2 elif angle_todo < -0.05: a = 3 elif angle_todo > +0.05: a = 1 return a def demo_heuristic_lander(env, seed=None, render=False): total_reward = 0 steps = 0 s = env.reset(seed=seed) while True: a = heuristic(env, s) s, r, done, info = env.step(a) total_reward += r if render: still_open = env.render() if still_open is False: break if steps % 20 == 0 or done: print("observations:", " ".join([f"{x:+0.2f}" for x in s])) print(f"step {steps} total_reward {total_reward:+0.2f}") steps += 1 if done: break if render: env.close() return total_reward class LunarLanderContinuous: def __init__(self): raise error.Error( "Error initializing LunarLanderContinuous Environment.\n" "Currently, we do not support initializing this mode of environment by calling the class directly.\n" "To use this environment, instead create it by specifying the continuous keyword in gym.make, i.e.\n" 'gym.make("LunarLander-v2", continuous=True)' ) if __name__ == "__main__": demo_heuristic_lander(LunarLander(), render=True)