Assistance Needed: Utilizing Pool Option in Emcee Sampler with Bilby
Dear Bilby team,
I am Manosh, and Bilby has been instrumental in conducting parameter estimation for my cosmological model. Although I am not well-versed in Python or gravitational waves, I have become a regular user of the Bilby library. Presently, I am utilizing Bilby version 2.1.1 with Python 3.8 for my research.
Recently, I encountered a challenge while attempting to employ the pool option in the emcee sampler for my analyses. Specifically, I faced difficulties when running the code provided at ( !341 (merged)).
I am reaching out to the Bilby community for guidance and support in understanding and resolving the issue related to the pool option. Any assistance or suggestions you could provide would be immensely valuable to me.
Additionally, as per the guidance of Dr. Matthew Pitkin, I tried using the npool option with the emcee sampler. It appears to have worked, as I observed the message "22:16 bilby INFO : Setting up multiprocessing pool with 8 processes" during execution, with all 8 cores active. However, the main error encountered was "NotImplementedError: pool objects cannot be passed between processes or pickled". For clarity, I have affixed the output.
Thank you for your time and support.
Best regards, Manosh
Manosh T M Department of Physics Cochin University of Science and Technology Kochi 682022, India
tm.manosh@gmail.com tm.manosh@cusat.ac.in manosh@cusat.ac.in manoshmanoharan.github.io ph: +91 9895346077 ph: +91 8075482248
22:16 bilby INFO : Running for label 'linear_regression_emcee', output will be saved to 'outdir_3'
22:16 bilby INFO : Analysis priors:
22:16 bilby INFO : m=Uniform(minimum=0, maximum=5, name='m', latex_label='m', unit=None, boundary=None)
22:16 bilby INFO : c=Uniform(minimum=-2, maximum=2, name='c', latex_label='c', unit=None, boundary=None)
22:16 bilby INFO : Analysis likelihood class: <class 'bilby.core.likelihood.GaussianLikelihood'>
22:16 bilby INFO : Analysis likelihood noise evidence: nan
22:16 bilby INFO : Single likelihood evaluation took 1.434e-04 s
22:16 bilby INFO : Using sampler Emcee with kwargs {'nwalkers': 500, 'a': 2, 'args': [], 'kwargs': {}, 'postargs': None, 'pool': None, 'live_dangerously': False, 'runtime_sortingfn': None, 'lnprob0': None, 'rstate0': None, 'blobs0': None, 'iterations': 200, 'thin': 1, 'storechain': True, 'mh_proposal': None}
22:16 bilby INFO : Setting up multiproccesing pool with 8 processes
100%|███████████████████████████████████████████| 200/200
[00:23<00:00, 8.57it/s]
22:16 bilby INFO : Checkpointing sampler to file outdir_3/emcee_linear_regression_emcee/sampler.pickle
22:16 bilby INFO : Max autocorr time = 22
22:16 bilby INFO : Discarding 100 steps for burn-in
22:16 bilby INFO : Sampling time: 0:00:23.643084
TypeError Traceback (most recent call last)
File ~/.local/lib/python3.8/site-packages/bilby/core/result.py:790, in Result.save_to_file(self, filename, overwrite, outdir, extension,
gzip)
789 with open(filename, 'w') as file:
--> 790 json.dump(dictionary, file, indent=2, cls=BilbyJsonEncoder)
791 elif extension == 'hdf5':
File /usr/lib64/python3.8/json/__init__.py:179, in dump(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
177 # could accelerate with writelines in some versions of Python, at
178 # a debuggability
cost
--> 179 for chunk in iterable:
180 fp.write(chunk)
File /usr/lib64/python3.8/json/encoder.py:431, in _make_iterencode.<locals>._iterencode(o, _current_indent_level)
430 elif isinstance(o, dict):
--> 431 yield from _iterencode_dict(o,_current_indent_level)
432 else:
File /usr/lib64/python3.8/json/encoder.py:405, in _make_iterencode.<locals>._iterencode_dict(dct, _current_indent_level)
404 chunks = _iterencode(value, _current_indent_level)
--> 405 yield from chunks
406 if newline_indent is not None:
File /usr/lib64/python3.8/json/encoder.py:405, in _make_iterencode.<locals>._iterencode_dict(dct, _current_indent_level)
404 chunks = _iterencode(value, _current_indent_level)
--> 405 yield from chunks
406 if newline_indent is not None:
File /usr/lib64/python3.8/json/encoder.py:438, in _make_iterencode.<locals>._iterencode(o, _current_indent_level)
437 markers[markerid] = o
--> 438 o = _default(o)
439 yield from _iterencode(o, _current_indent_level)
File ~/.local/lib/python3.8/site-packages/bilby/core/utils/io.py:87, in BilbyJsonEncoder.default(self, obj)
86 return obj.isoformat()
---> 87 return json.JSONEncoder.default(self, obj)
File /usr/lib64/python3.8/json/encoder.py:179, in JSONEncoder.default(self, o)
161 """Implement this method in a subclass such that it returns
162 a serializable object for ``o``, or calls the base implementation
163 (to raise a ``TypeError``).
(...)
177 178 """
--> 179 raise TypeError(f'Object of type {o.__class__.__name__} '
180 f'is not JSON serializable')
TypeError: Object of type Pool is not JSON serializable
During handling of the above exception, another exception occurred:
NotImplementedError Traceback (most recent call last)
Cell In[3], line 40
37 pool = Pool(2)
39 # And run sampler
---> 40 result = bilby.run_sampler(
41 likelihood=likelihood, priors=priors, sampler='emcee', nburn=100, nsteps=200,
42 npool=8, injection_parameters=injection_parameters, outdir=outdir,
43 label=label)
File ~/.local/lib/python3.8/site-packages/bilby/core/sampler/__init__.py:262, in run_sampler(likelihood, priors, label, outdir, sampler, use_ratio, injection_parameters, conversion_function, plot, default_priors_file, clean, meta_data, save, gzip, result_class, npool, **kwargs)
260 # Initial save of the sampler in case of failure in
samples_to_posterior
261 if save:
--> 262 result.save_to_file(extension=save, gzip=gzip, outdir=outdir)
264 if None not in [result.injection_parameters, conversion_function]:
265 result.injection_parameters = conversion_function(result.injection_parameters)
File ~/.local/lib/python3.8/site-packages/bilby/core/result.py:803, in Result.save_to_file(self, filename, overwrite, outdir, extension, gzip)
801 except Exception as e:
802 filename =
".".join(filename.split(".")[:-1]) + ".pkl"
--> 803
safe_file_dump(self, filename, "dill")
804 logger.error(
805 "\n\nSaving the data has failed with the following message:\n"
806 "{}\nData has been dumped to {}.\n\n".format(e, filename)
807 )
File ~/.local/lib/python3.8/site-packages/bilby/core/utils/io.py:385, in safe_file_dump(data, filename, module)
383 temp_filename = filename + ".temp"
384 with open(temp_filename, "wb") as file:
--> 385 module.dump(data, file) 386 shutil.move(temp_filename, filename)
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:235, in dump(obj, file, protocol, byref, fmode, recurse, **kwds)
233 _kwds = kwds.copy()
234 _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
--> 235 Pickler(file, protocol, **_kwds).dump(obj)
236 return
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:394, in Pickler.dump(self, obj) 392 def dump(self, obj): #NOTE: if settings change, need to update attributes
393 logger.trace_setup(self)
--> 394 StockPickler.dump(self, obj)
File /usr/lib64/python3.8/pickle.py:485, in _Pickler.dump(self, obj)
483 if self.proto >= 4:
484 self.framer.start_framing()
--> 485 self.save(obj)
486 self.write(STOP)
487 self.framer.end_framing()
File ~/.local/lib/python3.8/site-packages/dill/_dill.py: 388, in Pickler.save(self, obj, save_persistent_id)
386 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
387 raise PicklingError(msg)--> 388 StockPickler.save(self, obj, save_persistent_id)
File /usr/lib64/python3.8/pickle.py:601, in _Pickler.save(self, obj, save_persistent_id)
597 raise PicklingError("Tuple returned by %s must have "
598 "two to six elements" % reduce)
600 # Save the reduce() output and finally memoize the object
--> 601 self.save_reduce(obj=obj, *rv)
File /usr/lib64/python3.8/pickle.py:715, in _Pickler.save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
713 if state is not None:
714 if state_setter is None:
--> 715. save(state)
716 write(BUILD)
717 else:
718 # If a state_setter is specified, call it instead of load_build
719 # to update obj's with its previous state.
720 # First, push state_setter and its tuple of expected arguments
721 # (obj, state) onto the stack.
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:388, in Pickler.save(self, obj, save_persistent_id)
386 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
387 raise PicklingError(msg)--> 388 StockPickler.save(self, obj, save_persistent_id)
File /usr/lib64/python3.8/pickle.py:558, in _Pickler.save(self, obj, save_persistent_id)
556 f = self.dispatch.get(t)
557 if f is not None:
--> 558 f(self, obj) # Call unbound method with explicit self
559 return
561 # Check private dispatch table if any, or else
562 # copyreg.dispatch_table
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:1186, in save_module_dict(pickler, obj)
1183 if is_dill(pickler, child=False) and pickler._session:
1184 # we only care about session the first pass thru
1185 pickler._first_pass = False
--> 1186 StockPickler.save_dict(pickler, obj)
1187 logger.trace(pickler, "# D2") 1188 return
File /usr/lib64/python3.8/pickle.py:969, in _Pickler.save_dict(self, obj)
966 self.write(MARK + DICT)
968 self.memoize(obj)
--> 969 self._batch_setitems(obj.items())
File /usr/lib64/python3.8/pickle.py:995, in _Pickler._batch_setitems(self, items)
993 for k, v in tmp:
994 save(k)
--> 995 save(v)
996 write(SETITEMS)
997 elif n:
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:388, in Pickler.save(self, obj, save_persistent_id)
386 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
387 raise PicklingError(msg)
--> 388 StockPickler.save(self, obj, save_persistent_id)
File /usr/lib64/python3.8/pickle.py:558, in _Pickler.save(self, obj, save_persistent_id)
556 f = self.dispatch.get(t)
557 if f is not None:
--> 558 f(self, obj) # Call unbound method with explicit self
559 return
561 # Check private dispatch table if any, or else
562 # copyreg.dispatch_table
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:1186, in save_module_dict(pickler, obj)
1183 if is_dill(pickler, child=False) and pickler._session:
1184 # we only care about session the first pass thru
1185 pickler._first_pass = False
--> 1186 StockPickler.save_dict(pickler, obj)
1187 logger.trace(pickler, "# D2")
1188 return
File /usr/lib64/python3.8/pickle.py:969, in _Pickler.save_dict(self, obj)
966 self.write(MARK + DICT) 968 self.memoize(obj)
--> 969 self._batch_setitems(obj.items())
File /usr/lib64/python3.8/pickle.py:995, in _Pickler._batch_setitems(self, items)
993 for k, v in tmp:
994 save(k)
--> 995 save(v)
996 write(SETITEMS)
997 elif n:
File ~/.local/lib/python3.8/site-packages/dill/_dill.py:388, in Pickler.save(self, obj, save_persistent_id)
386 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
387 raise PicklingError(msg)
--> 388 StockPickler.save(self, obj, save_persistent_id)
File /usr/lib64/python3.8/pickle.py:576, in _Pickler.save(self, obj, save_persistent_id)
574 reduce = getattr(obj, "__reduce_ex__", None)
575 if reduce is not None:
--> 576 rv = reduce(self.proto)
577 else:
578 reduce = getattr(obj, "__reduce__", None)
File /usr/lib64/python3.8/multiprocessing/pool.py:640, in Pool.__reduce__(self)
639 def __reduce__(self):
--> 640 raise NotImplementedError(
641 'pool objects cannot be passed between processes or pickled'
642 )
NotImplementedError: pool objects cannot be passed between processes or pickled