Update documentation build tooling (#700)

Co-authored-by: Martin van Hensbergen <martin@mvhensbergen.com>

docs/README.md

@@ -47,9 +47,11 @@ To rebuild the documentation automatically every time a change is made:
 
 ```
 cd docs
-sphinx-autobuild -b dirhtml . _build
+sphinx-autobuild -b dirhtml --watch ../gymnasium --re-ignore "pickle$" . _build
 ```
 
+You can then open http://localhost:8000 in your browser to watch a live updated version of the documentation.
+
 ## Writing Tutorials
 
 We use Sphinx-Gallery to build the tutorials inside the `docs/tutorials` directory. Check `docs/tutorials/demo.py` to see an example of a tutorial and [Sphinx-Gallery documentation](https://sphinx-gallery.github.io/stable/syntax.html) for more information.

docs/conf.py

@@ -8,19 +8,21 @@
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+# documentation root, use os.path.abspath to make it absolute.
 
 # -- Project information -----------------------------------------------------
 import os
 import re
+import sys
 
 import sphinx_gallery.gen_rst
 
-import gymnasium
+
+# Path setup for building from source tree
+sys.path.insert(0, os.path.abspath("."))  # For building from root
+sys.path.insert(0, os.path.abspath(".."))  # For building from docs dir
+
+import gymnasium  # noqa: E402
 
 
 project = "Gymnasium"