Gymnasium/gym/envs/unittest/cube_crash.py

import numpy as np
import gym
from gym import spaces
from gym.utils import seeding

# Unit test environment for CNNs and CNN+RNN algorithms.
# Looks like this (RGB observations):
#
#  ---------------------------
# |                           |
# |                           |
# |                           |
# |          **               |
# |          **               |
# |                           |
# |                           |
# |                           |
# |                           |
# |                           |
#  ========     ==============
#
# Goal is to go through the hole at the bottom. Agent controls square using Left-Nop-Right actions.
# It falls down automatically, episode length is a bit less than FIELD_H
#
# CubeCrash-v0                    # shaped reward
# CubeCrashSparse-v0              # reward 0 or 1 at the end
# CubeCrashScreenBecomesBlack-v0  # for RNNs
#
# To see how it works, run:
#
# python examples/agents/keyboard_agent.py CubeCrashScreen-v0

FIELD_W = 32
FIELD_H = 40
HOLE_WIDTH = 8

color_black = np.array((0,0,0)).astype('float32')
color_white = np.array((255,255,255)).astype('float32')
color_green = np.array((0,255,0)).astype('float32')

class CubeCrash(gym.Env):
    metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second' : 60,
        'video.res_w' : FIELD_W,
        'video.res_h' : FIELD_H,
    }

    use_shaped_reward = True
    use_black_screen  = False
    use_random_colors = False   # Makes env too hard

    def __init__(self):
        self.seed()
        self.viewer = None

        self.observation_space = spaces.Box(0, 255, (FIELD_H,FIELD_W,3), dtype=np.uint8)
        self.action_space = spaces.Discrete(3)

        self.reset()

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def random_color(self):
        return np.array([
            self.np_random.randint(low=0, high=255),
            self.np_random.randint(low=0, high=255),
            self.np_random.randint(low=0, high=255),
            ]).astype('uint8')

    def reset(self):
        self.cube_x = self.np_random.randint(low=3, high=FIELD_W-3)
        self.cube_y = self.np_random.randint(low=3, high=FIELD_H//6)
        self.hole_x = self.np_random.randint(low=HOLE_WIDTH, high=FIELD_W-HOLE_WIDTH)
        self.bg_color = self.random_color() if self.use_random_colors else color_black
        self.potential  = None
        self.step_n = 0
        while 1:
            self.wall_color = self.random_color() if self.use_random_colors else color_white
            self.cube_color = self.random_color() if self.use_random_colors else color_green
            if np.linalg.norm(self.wall_color - self.bg_color) < 50 or np.linalg.norm(self.cube_color - self.bg_color) < 50: continue
            break
        return self.step(0)[0]

    def step(self, action):
        if action==0: pass
        elif action==1: self.cube_x -= 1
        elif action==2: self.cube_x += 1
        else: assert 0, "Action %i is out of range" % action
        self.cube_y += 1
        self.step_n += 1

        obs = np.zeros( (FIELD_H,FIELD_W,3), dtype=np.uint8 )
        obs[:,:,:] = self.bg_color
        obs[FIELD_H-5:FIELD_H,:,:] = self.wall_color
        obs[FIELD_H-5:FIELD_H, self.hole_x-HOLE_WIDTH//2:self.hole_x+HOLE_WIDTH//2+1, :] = self.bg_color
        obs[self.cube_y-1:self.cube_y+2, self.cube_x-1:self.cube_x+2, :] = self.cube_color
        if self.use_black_screen and self.step_n > 4:
            obs[:] = np.zeros((3,), dtype=np.uint8)

        done = False
        reward = 0
        dist = np.abs(self.cube_x - self.hole_x)
        if self.potential is not None and self.use_shaped_reward:
            reward = (self.potential - dist) * 0.01
        self.potential = dist

        if self.cube_x-1 < 0 or self.cube_x+1 >= FIELD_W:
            done = True
            reward = -1
        elif self.cube_y+1 >= FIELD_H-5:
            if dist >= HOLE_WIDTH//2:
                done = True
                reward = -1
            elif self.cube_y == FIELD_H:
                done = True
                reward = +1
        self.last_obs = obs
        return obs, reward, done, {}

    def render(self, mode='human'):
        if mode == 'rgb_array':
            return self.last_obs

        elif mode == 'human':
            from gym.envs.classic_control import rendering
            if self.viewer is None:
                self.viewer = rendering.SimpleImageViewer()
            self.viewer.imshow(self.last_obs)
            return self.viewer.isopen

        else:
            assert 0, "Render mode '%s' is not supported" % mode

    def close(self):
        if self.viewer is not None:
            self.viewer.close()
            self.viewer = None

class CubeCrashSparse(CubeCrash):
    use_shaped_reward = False

class CubeCrashScreenBecomesBlack(CubeCrash):
    use_shaped_reward = False
    use_black_screen = True