diff --git a/gym/envs/robotics/fetch/pick_and_place.py b/gym/envs/robotics/fetch/pick_and_place.py
index 9b4d049d0..c6c5e7ea9 100644
--- a/gym/envs/robotics/fetch/pick_and_place.py
+++ b/gym/envs/robotics/fetch/pick_and_place.py
@@ -1,7 +1,12 @@
+import os
 from gym import utils
 from gym.envs.robotics import fetch_env
 
 
+# Ensure we get the path separator correct on windows
+MODEL_XML_PATH = os.path.join('fetch', 'pick_and_place.xml')
+
+
 class FetchPickAndPlaceEnv(fetch_env.FetchEnv, utils.EzPickle):
     def __init__(self, reward_type='sparse'):
         initial_qpos = {
@@ -11,7 +16,7 @@ class FetchPickAndPlaceEnv(fetch_env.FetchEnv, utils.EzPickle):
             'object0:joint': [1.25, 0.53, 0.4, 1., 0., 0., 0.],
         }
         fetch_env.FetchEnv.__init__(
-            self, 'fetch/pick_and_place.xml', has_object=True, block_gripper=False, n_substeps=20,
+            self, MODEL_XML_PATH, has_object=True, block_gripper=False, n_substeps=20,
             gripper_extra_height=0.2, target_in_the_air=True, target_offset=0.0,
             obj_range=0.15, target_range=0.15, distance_threshold=0.05,
             initial_qpos=initial_qpos, reward_type=reward_type)
diff --git a/gym/envs/robotics/fetch/push.py b/gym/envs/robotics/fetch/push.py
index a3c19633b..bde15ec00 100644
--- a/gym/envs/robotics/fetch/push.py
+++ b/gym/envs/robotics/fetch/push.py
@@ -1,7 +1,12 @@
+import os
 from gym import utils
 from gym.envs.robotics import fetch_env
 
 
+# Ensure we get the path separator correct on windows
+MODEL_XML_PATH = os.path.join('fetch', 'push.xml')
+
+
 class FetchPushEnv(fetch_env.FetchEnv, utils.EzPickle):
     def __init__(self, reward_type='sparse'):
         initial_qpos = {
@@ -11,7 +16,7 @@ class FetchPushEnv(fetch_env.FetchEnv, utils.EzPickle):
             'object0:joint': [1.25, 0.53, 0.4, 1., 0., 0., 0.],
         }
         fetch_env.FetchEnv.__init__(
-            self, 'fetch/push.xml', has_object=True, block_gripper=True, n_substeps=20,
+            self, MODEL_XML_PATH, has_object=True, block_gripper=True, n_substeps=20,
             gripper_extra_height=0.0, target_in_the_air=False, target_offset=0.0,
             obj_range=0.15, target_range=0.15, distance_threshold=0.05,
             initial_qpos=initial_qpos, reward_type=reward_type)
diff --git a/gym/envs/robotics/fetch/reach.py b/gym/envs/robotics/fetch/reach.py
index 7504e4dad..cc3fc46c6 100644
--- a/gym/envs/robotics/fetch/reach.py
+++ b/gym/envs/robotics/fetch/reach.py
@@ -1,7 +1,12 @@
+import os
 from gym import utils
 from gym.envs.robotics import fetch_env
 
 
+# Ensure we get the path separator correct on windows
+MODEL_XML_PATH = os.path.join('fetch', 'reach.xml')
+
+
 class FetchReachEnv(fetch_env.FetchEnv, utils.EzPickle):
     def __init__(self, reward_type='sparse'):
         initial_qpos = {
@@ -10,7 +15,7 @@ class FetchReachEnv(fetch_env.FetchEnv, utils.EzPickle):
             'robot0:slide2': 0.0,
         }
         fetch_env.FetchEnv.__init__(
-            self, 'fetch/reach.xml', has_object=False, block_gripper=True, n_substeps=20,
+            self, MODEL_XML_PATH, has_object=False, block_gripper=True, n_substeps=20,
             gripper_extra_height=0.2, target_in_the_air=True, target_offset=0.0,
             obj_range=0.15, target_range=0.15, distance_threshold=0.05,
             initial_qpos=initial_qpos, reward_type=reward_type)
diff --git a/gym/envs/robotics/fetch/slide.py b/gym/envs/robotics/fetch/slide.py
index b8a02a9a3..8c893b2b7 100644
--- a/gym/envs/robotics/fetch/slide.py
+++ b/gym/envs/robotics/fetch/slide.py
@@ -1,9 +1,14 @@
+import os
 import numpy as np
 
 from gym import utils
 from gym.envs.robotics import fetch_env
 
 
+# Ensure we get the path separator correct on windows
+MODEL_XML_PATH = os.path.join('fetch', 'slide.xml')
+
+
 class FetchSlideEnv(fetch_env.FetchEnv, utils.EzPickle):
     def __init__(self, reward_type='sparse'):
         initial_qpos = {
@@ -13,7 +18,7 @@ class FetchSlideEnv(fetch_env.FetchEnv, utils.EzPickle):
             'object0:joint': [1.7, 1.1, 0.4, 1., 0., 0., 0.],
         }
         fetch_env.FetchEnv.__init__(
-            self, 'fetch/slide.xml', has_object=True, block_gripper=True, n_substeps=20,
+            self, MODEL_XML_PATH, has_object=True, block_gripper=True, n_substeps=20,
             gripper_extra_height=-0.02, target_in_the_air=False, target_offset=np.array([0.4, 0.0, 0.0]),
             obj_range=0.1, target_range=0.3, distance_threshold=0.05,
             initial_qpos=initial_qpos, reward_type=reward_type)
diff --git a/gym/envs/robotics/hand/manipulate.py b/gym/envs/robotics/hand/manipulate.py
index e488d49d9..de55f3482 100644
--- a/gym/envs/robotics/hand/manipulate.py
+++ b/gym/envs/robotics/hand/manipulate.py
@@ -1,3 +1,4 @@
+import os
 import numpy as np
 
 from gym import utils, error
@@ -18,6 +19,12 @@ def quat_from_angle_and_axis(angle, axis):
     return quat
 
 
+# Ensure we get the path separator correct on windows
+MANIPULATE_BLOCK_XML = os.path.join('hand', 'manipulate_block.xml')
+MANIPULATE_EGG_XML = os.path.join('hand', 'manipulate_egg.xml')
+MANIPULATE_PEN_XML = os.path.join('hand', 'manipulate_pen.xml')
+
+
 class ManipulateEnv(hand_env.HandEnv, utils.EzPickle):
     def __init__(
         self, model_path, target_position, target_rotation,
@@ -267,7 +274,7 @@ class ManipulateEnv(hand_env.HandEnv, utils.EzPickle):
 class HandBlockEnv(ManipulateEnv):
     def __init__(self, target_position='random', target_rotation='xyz', reward_type='sparse'):
         super(HandBlockEnv, self).__init__(
-            model_path='hand/manipulate_block.xml', target_position=target_position,
+            model_path=MANIPULATE_BLOCK_XML, target_position=target_position,
             target_rotation=target_rotation,
             target_position_range=np.array([(-0.04, 0.04), (-0.06, 0.02), (0.0, 0.06)]),
             reward_type=reward_type)
@@ -276,7 +283,7 @@ class HandBlockEnv(ManipulateEnv):
 class HandEggEnv(ManipulateEnv):
     def __init__(self, target_position='random', target_rotation='xyz', reward_type='sparse'):
         super(HandEggEnv, self).__init__(
-            model_path='hand/manipulate_egg.xml', target_position=target_position,
+            model_path=MANIPULATE_EGG_XML, target_position=target_position,
             target_rotation=target_rotation,
             target_position_range=np.array([(-0.04, 0.04), (-0.06, 0.02), (0.0, 0.06)]),
             reward_type=reward_type)
@@ -285,7 +292,7 @@ class HandEggEnv(ManipulateEnv):
 class HandPenEnv(ManipulateEnv):
     def __init__(self, target_position='random', target_rotation='xyz', reward_type='sparse'):
         super(HandPenEnv, self).__init__(
-            model_path='hand/manipulate_pen.xml', target_position=target_position,
+            model_path=MANIPULATE_PEN_XML, target_position=target_position,
             target_rotation=target_rotation,
             target_position_range=np.array([(-0.04, 0.04), (-0.06, 0.02), (0.0, 0.06)]),
             randomize_initial_rotation=False, reward_type=reward_type,
diff --git a/gym/envs/robotics/hand/reach.py b/gym/envs/robotics/hand/reach.py
index 1f6f9f8ed..81ed9f954 100644
--- a/gym/envs/robotics/hand/reach.py
+++ b/gym/envs/robotics/hand/reach.py
@@ -1,3 +1,4 @@
+import os
 import numpy as np
 
 from gym import utils
@@ -42,6 +43,10 @@ DEFAULT_INITIAL_QPOS = {
 }
 
 
+# Ensure we get the path separator correct on windows
+MODEL_XML_PATH = os.path.join('hand', 'reach.xml')
+
+
 def goal_distance(goal_a, goal_b):
     assert goal_a.shape == goal_b.shape
     return np.linalg.norm(goal_a - goal_b, axis=-1)
@@ -56,7 +61,7 @@ class HandReachEnv(hand_env.HandEnv, utils.EzPickle):
         self.reward_type = reward_type
 
         hand_env.HandEnv.__init__(
-            self, 'hand/reach.xml', n_substeps=n_substeps, initial_qpos=initial_qpos,
+            self, MODEL_XML_PATH, n_substeps=n_substeps, initial_qpos=initial_qpos,
             relative_control=relative_control)
         utils.EzPickle.__init__(self)