2019-06-21 17:29:44 -04:00
|
|
|
import contextlib
|
|
|
|
import os
|
|
|
|
|
2021-07-29 02:26:34 +02:00
|
|
|
__all__ = ["CloudpickleWrapper", "clear_mpi_env_vars"]
|
|
|
|
|
2019-06-21 17:29:44 -04:00
|
|
|
|
|
|
|
class CloudpickleWrapper(object):
|
|
|
|
def __init__(self, fn):
|
|
|
|
self.fn = fn
|
|
|
|
|
|
|
|
def __getstate__(self):
|
|
|
|
import cloudpickle
|
2021-07-29 02:26:34 +02:00
|
|
|
|
2019-06-21 17:29:44 -04:00
|
|
|
return cloudpickle.dumps(self.fn)
|
|
|
|
|
|
|
|
def __setstate__(self, ob):
|
|
|
|
import pickle
|
2021-07-29 02:26:34 +02:00
|
|
|
|
2019-06-21 17:29:44 -04:00
|
|
|
self.fn = pickle.loads(ob)
|
|
|
|
|
|
|
|
def __call__(self):
|
|
|
|
return self.fn()
|
|
|
|
|
2021-07-29 02:26:34 +02:00
|
|
|
|
2019-06-21 17:29:44 -04:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def clear_mpi_env_vars():
|
|
|
|
"""
|
|
|
|
`from mpi4py import MPI` will call `MPI_Init` by default. If the child
|
|
|
|
process has MPI environment variables, MPI will think that the child process
|
|
|
|
is an MPI process just like the parent and do bad things such as hang.
|
2021-07-29 02:26:34 +02:00
|
|
|
|
2019-06-21 17:29:44 -04:00
|
|
|
This context manager is a hacky way to clear those environment variables
|
|
|
|
temporarily such as when we are starting multiprocessing Processes.
|
|
|
|
"""
|
|
|
|
removed_environment = {}
|
|
|
|
for k, v in list(os.environ.items()):
|
2021-07-29 02:26:34 +02:00
|
|
|
for prefix in ["OMPI_", "PMI_"]:
|
2019-06-21 17:29:44 -04:00
|
|
|
if k.startswith(prefix):
|
|
|
|
removed_environment[k] = v
|
|
|
|
del os.environ[k]
|
|
|
|
try:
|
|
|
|
yield
|
|
|
|
finally:
|
|
|
|
os.environ.update(removed_environment)
|