Test necessary imports (#146)

.github/workflows/build.yml

@@ -5,7 +5,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
-  build:
+  build-all:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -13,8 +13,21 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: |
-           docker build -f py.Dockerfile \
+           docker build -f bin/all-py.Dockerfile \
              --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
-             --tag gymnasium-docker .
+             --tag gymnasium-all-docker .
       - name: Run tests
-        run: docker run gymnasium-docker pytest
+        run: docker run gymnasium-all-docker pytest tests/*
+
+  build-necessary:
+    runs-on:
+      ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+           docker build -f bin/necessary-py.Dockerfile \
+             --build-arg PYTHON_VERSION='3.10' \
+             --tag gymnasium-necessary-docker .
+      - name: Run tests
+        run: |
+           docker run gymnasium-necessary-docker pytest tests/test_core.py tests/envs/test_compatibility.py tests/envs/test_envs.py tests/spaces

bin/all-py.Dockerfile

@@ -26,6 +26,6 @@ ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/root/.mujoco/mujoco210/bin"
 COPY . /usr/local/gymnasium/
 WORKDIR /usr/local/gymnasium/
 
-RUN pip install .[testing] --no-cache-dir
+RUN pip install .[all,testing] --no-cache-dir
 
 ENTRYPOINT ["/usr/local/gymnasium/bin/docker_entrypoint"]

bin/necessary-py.Dockerfile

@@ -0,0 +1,25 @@
+# A Dockerfile that sets up a full Gymnasium install with test dependencies
+ARG PYTHON_VERSION
+FROM python:$PYTHON_VERSION
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN apt-get -y update \
+    && apt-get install --no-install-recommends -y \
+    unzip \
+    libglu1-mesa-dev \
+    libgl1-mesa-dev \
+    libosmesa6-dev \
+    xvfb \
+    patchelf \
+    ffmpeg cmake \
+    && apt-get autoremove -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /usr/local/gymnasium/
+WORKDIR /usr/local/gymnasium/
+
+RUN pip install .[testing] --no-cache-dir
+
+ENTRYPOINT ["/usr/local/gymnasium/bin/docker_entrypoint"]

setup.py

@@ -1,5 +1,6 @@
 """Setups the project."""
 import itertools
+from typing import Dict, List
 
 from setuptools import find_packages, setup
 
@@ -32,7 +33,7 @@ def get_version():
 
 
 # Environment-specific dependencies.
-extras = {
+extras: Dict[str, List[str]] = {
     "atari": ["shimmy[atari]>=0.1.0,<1.0"],
     "accept-rom-license": ["autorom[accept-rom-license]~=0.4.2"],
     "box2d": ["box2d-py==2.3.5", "pygame==2.1.0", "swig==4.*"],
@@ -44,15 +45,14 @@ extras = {
     "other": ["lz4>=3.1.0", "opencv-python>=3.0", "matplotlib>=3.0", "moviepy>=1.0.0"],
 }
 
-extras["testing"] = list(set(itertools.chain.from_iterable(extras.values()))) + [
-    "pytest==7.1.3",
-]
-
 # All dependency groups - accept rom license as requires user to run
 all_groups = set(extras.keys()) - {"accept-rom-license"}
 extras["all"] = list(
     set(itertools.chain.from_iterable(map(lambda group: extras[group], all_groups)))
 )
+extras["testing"] = [
+    "pytest==7.1.3",
+]
 
 version = get_version()
 header_count, long_description = get_description()

tests/envs/test_envs.py

@@ -1,12 +1,10 @@
 import pickle
 import warnings
 
-import numpy as np
 import pytest
 
 import gymnasium as gym
 from gymnasium.envs.registration import EnvSpec
-from gymnasium.logger import warn
 from gymnasium.utils.env_checker import check_env, data_equivalence
 from tests.envs.utils import (
     all_testing_env_specs,
@@ -44,7 +42,7 @@ def test_envs_pass_env_checker(spec):
     """Check that all environments pass the environment checker with no warnings other than the expected."""
     with warnings.catch_warnings(record=True) as caught_warnings:
         env = spec.make(disable_env_checker=True).unwrapped
-        check_env(env)
+        check_env(env, skip_render_check=True)
 
         env.close()
 
@@ -119,75 +117,6 @@ def test_env_determinism_rollout(env_spec: EnvSpec):
     env_2.close()
 
 
-def check_rendered(rendered_frame, mode: str):
-    """Check that the rendered frame is as expected."""
-    if mode == "rgb_array_list":
-        assert isinstance(rendered_frame, list)
-        for frame in rendered_frame:
-            check_rendered(frame, "rgb_array")
-    elif mode == "rgb_array":
-        assert isinstance(rendered_frame, np.ndarray)
-        assert len(rendered_frame.shape) == 3
-        assert rendered_frame.shape[2] == 3
-        assert np.all(rendered_frame >= 0) and np.all(rendered_frame <= 255)
-    elif mode == "ansi":
-        assert isinstance(rendered_frame, str)
-        assert len(rendered_frame) > 0
-    elif mode == "state_pixels_list":
-        assert isinstance(rendered_frame, list)
-        for frame in rendered_frame:
-            check_rendered(frame, "rgb_array")
-    elif mode == "state_pixels":
-        check_rendered(rendered_frame, "rgb_array")
-    elif mode == "depth_array_list":
-        assert isinstance(rendered_frame, list)
-        for frame in rendered_frame:
-            check_rendered(frame, "depth_array")
-    elif mode == "depth_array":
-        assert isinstance(rendered_frame, np.ndarray)
-        assert len(rendered_frame.shape) == 2
-    else:
-        warn(
-            f"Unknown render mode: {mode}, cannot check that the rendered data is correct. Add case to `check_rendered`"
-        )
-
-
-# We do not check render_mode for some mujoco envs and any old Gym environment wrapped by `GymEnvironment`
-render_mode_env_specs = [
-    spec
-    for spec in all_testing_env_specs
-    if "mujoco" not in spec.entry_point or "v4" in spec.id
-]
-
-
-@pytest.mark.parametrize(
-    "spec", render_mode_env_specs, ids=[spec.id for spec in render_mode_env_specs]
-)
-def test_render_modes(spec):
-    """There is a known issue where rendering a mujoco environment then mujoco-py will cause an error on non-mac based systems.
-
-    Therefore, we are only testing with mujoco environments.
-    """
-    env = spec.make()
-
-    assert "rgb_array" in env.metadata["render_modes"]
-
-    for mode in env.metadata["render_modes"]:
-        if mode != "human":
-            new_env = spec.make(render_mode=mode)
-
-            new_env.reset()
-            rendered = new_env.render()
-            check_rendered(rendered, mode)
-
-            new_env.step(new_env.action_space.sample())
-            rendered = new_env.render()
-            check_rendered(rendered, mode)
-
-            new_env.close()
-    env.close()
-
-
 @pytest.mark.parametrize(
     "env",
     all_testing_initialised_envs,

tests/envs/test_rendering.py

@@ -0,0 +1,74 @@
+import numpy as np
+import pytest
+
+from gymnasium.logger import warn
+from tests.envs.utils import all_testing_env_specs
+
+
+def check_rendered(rendered_frame, mode: str):
+    """Check that the rendered frame is as expected."""
+    if mode == "rgb_array_list":
+        assert isinstance(rendered_frame, list)
+        for frame in rendered_frame:
+            check_rendered(frame, "rgb_array")
+    elif mode == "rgb_array":
+        assert isinstance(rendered_frame, np.ndarray)
+        assert len(rendered_frame.shape) == 3
+        assert rendered_frame.shape[2] == 3
+        assert np.all(rendered_frame >= 0) and np.all(rendered_frame <= 255)
+    elif mode == "ansi":
+        assert isinstance(rendered_frame, str)
+        assert len(rendered_frame) > 0
+    elif mode == "state_pixels_list":
+        assert isinstance(rendered_frame, list)
+        for frame in rendered_frame:
+            check_rendered(frame, "rgb_array")
+    elif mode == "state_pixels":
+        check_rendered(rendered_frame, "rgb_array")
+    elif mode == "depth_array_list":
+        assert isinstance(rendered_frame, list)
+        for frame in rendered_frame:
+            check_rendered(frame, "depth_array")
+    elif mode == "depth_array":
+        assert isinstance(rendered_frame, np.ndarray)
+        assert len(rendered_frame.shape) == 2
+    else:
+        warn(
+            f"Unknown render mode: {mode}, cannot check that the rendered data is correct. Add case to `check_rendered`"
+        )
+
+
+# We do not check render_mode for some mujoco envs and any old Gym environment wrapped by `GymEnvironment`
+render_mode_env_specs = [
+    spec
+    for spec in all_testing_env_specs
+    if "mujoco" not in spec.entry_point or "v4" in spec.id
+]
+
+
+@pytest.mark.parametrize(
+    "spec", render_mode_env_specs, ids=[spec.id for spec in render_mode_env_specs]
+)
+def test_render_modes(spec):
+    """There is a known issue where rendering a mujoco environment then mujoco-py will cause an error on non-mac based systems.
+
+    Therefore, we are only testing with mujoco environments.
+    """
+    env = spec.make()
+
+    assert "rgb_array" in env.metadata["render_modes"]
+
+    for mode in env.metadata["render_modes"]:
+        if mode != "human":
+            new_env = spec.make(render_mode=mode)
+
+            new_env.reset()
+            rendered = new_env.render()
+            check_rendered(rendered, mode)
+
+            new_env.step(new_env.action_space.sample())
+            rendered = new_env.render()
+            check_rendered(rendered, mode)
+
+            new_env.close()
+    env.close()

tests/envs/utils.py

@@ -51,7 +51,6 @@ gym_testing_env_specs: List[EnvSpec] = [
         for ep in ["box2d", "classic_control", "toy_text"]
     )
 ]
-# TODO, add minimum testing env spec in testing
 
 
 def assert_equals(a, b, prefix=None):