Merge branch 'master' into redshifted_prior

.gitlab-ci.yml

@@ -13,6 +13,31 @@ stages:
   - test
   - deploy
 
+.test-python: &test-python
+  stage: test
+  image: python
+  before_script:
+    # this is required because pytables doesn't use a wheel on py37
+    - apt-get -yqq update
+    - apt-get -yqq install libhdf5-dev
+  script:
+    - python -m pip install .
+    - python -c "import bilby"
+    - python -c "import bilby.core"
+    - python -c "import bilby.gw"
+    - python -c "import bilby.hyper"
+    - python -c "import cli_bilby"
+
+# test basic setup on python2
+basic-2.7:
+  <<: *test-python
+  image: python:2.7
+
+# test basic setup on python3
+basic-3.7:
+  <<: *test-python
+  image: python:3.7
+
 # test example on python 2
 python-2.7:
   stage: test
@@ -54,6 +79,28 @@ python-3.7:
       - coverage_badge.svg
       - docs/_build/html/
 
+# Tests run at a fixed schedule rather than on push
+scheduled-python-3.7:
+  stage: test
+  image: bilbydev/bilby-test-suite-python37
+  only:
+    - schedules
+  before_script:
+    # Install the dependencies specified in the Pipfile
+    - pipenv install --three --python=/opt/conda/bin/python --system --deploy
+  script:
+    - python setup.py install
+
+    # Run pyflakes
+    - flake8 .
+
+    # Run tests
+    - pytest
+
+    # Run tests which are only done on schedule
+    - pytest test/example_test.py
+    - pytest test/gw_example_test.py
+
 pages:
   stage: deploy
   dependencies:

CHANGELOG.md

@@ -3,15 +3,45 @@
 ## Unreleased
 
 ### Added
-- 
+- Support for JSON result files
+- Before sampling a test is performed for redundant priors
 
 ### Changed
+- Fixed the definition of iota to theta_jn. WARNING: this breaks backward compatibility. Previously, the CBC parameter iota was used in prior files, but was ill-defined. This fixes that, requiring all scripts to use `theta_jn` in place of `iota`
+- Changed the default result file store to JSON rather than hdf5. Reading/writing of hdf5 files is still intact. The read_in_result function will still read in hdf5 files for backward compatibility
+- Minor fixes to the way PSDs are calculated
+- Fixed a bug in the CBC result where the frequency_domain model was pickled
+- Use pickling to store the dynesty resume file and add a write-to-resume on SIGINT/SIGKILL
+- Bug fix in ROQ likelihood
+- Distance and phase marginalisation work with ROQ likelihood
+
+### Removed
+-
+
+## [0.4.0] 2019-02-15
+
+### Changed
+- Changed the logic around redundancy tests in the `PriorDict` classes
+- Fixed an accidental addition of astropy as a first-class dependency and added a check for missing dependencies to the C.I.
+- Fixed a bug in the "create-your-own-time-domain-model" example
+- Added citation guide to the readme
+
+## [0.3.6] 2019-02-10
+
+### Added
+- Added the PolyChord sampler, which can be accessed by using `sampler='pypolychord'` in `run_sampler`
+- `emcee` now writes all progress to disk and can resume from a previous run.
+
+### Changed
+- Cosmology generalised, users can now specify the cosmology used, default is astropy Planck15
+- UniformComovingVolume prior *requires* the name to be one of "luminosity_distance", "comoving_distance", "redshift"
 - Time/frequency array generation/conversion improved. We now impose `duration` is an integer multiple of
   `sampling_frequency`. Converting back and forth between time/frequency arrays now works for all valid arrays.
 - Updates the bilby.core.utils constants to match those of Astropy v3.0.4
+- Improve the load_data_from_cache_file method
 
 ### Removed
--
+- Removed deprecated `PriorSet` classes. Use `PriorDict` instead.
 
 ## [0.3.5] 2019-01-25
 

Pipfile

@@ -6,11 +6,12 @@ name = "pypi"
 [packages]
 future = "*"
 corner = "*"
-numpy = ">=1.9"
+numpy = "==1.15.2"
+ligotimegps = "<=1.2.3"
 matplotlib = "<3"
 scipy = ">=0.16"
-pandas = "*"
-deepdish = "*"
+pandas = "==0.23.0"
+deepdish = "==0.3.6"
 mock = "*"
 astropy = "<3"
 gwpy = "*"
@@ -23,7 +24,6 @@ emcee = "*"
 nestle = "*"
 ptemcee = "*"
 pymc3 = "*"
-pymultinest = "*"
 
 [requires]
 

README.rst

 |pipeline status| |coverage report| |pypi| |conda| |version|
 
+=====
 Bilby
 =====
 
@@ -23,6 +24,46 @@ help in creating a merge request, see `this page
 <https://docs.gitlab.com/ee/gitlab-basics/add-merge-request.html>`__ or contact
 us directly. For advice on contributing, see `this help page <https://git.ligo.org/lscsoft/bilby/blob/master/CONTRIBUTING.md>`__.
 
+
+--------------
+Citation guide
+--------------
+
+If you use :code:`bilby` in a scientific publication, please cite
+
+* `Bilby: A user-friendly Bayesian inference library for gravitational-wave
+  astronomy
+  <https://ui.adsabs.harvard.edu/#abs/2018arXiv181102042A/abstract>`__
+
+Additionally, :code:`bilby` builds on a number of open-source packages. If you
+make use of this functionality in your publications, we recommend you cite them
+as requested in their associated documentation.
+
+**Samplers**
+
+* `dynesty <https://github.com/joshspeagle/dynesty>`__
+* `nestle <https://github.com/kbarbary/nestle>`__
+* `pymultinest <https://github.com/JohannesBuchner/PyMultiNest>`__
+* `cpnest <https://github.com/johnveitch/cpnest>`__
+* `emcee <https://github.com/dfm/emcee>`__
+* `ptemcee <https://github.com/willvousden/ptemcee>`__
+* `ptmcmcsampler <https://github.com/jellis18/PTMCMCSampler>`__
+* `pypolychord <https://github.com/vhaasteren/pypolychord>`__
+* `PyMC3 <https://github.com/pymc-devs/pymc3>`_
+
+
+**Gravitational-wave tools**
+
+* `gwpy <https://github.com/gwpy/gwpy>`__
+* `lalsuite <https://git.ligo.org/lscsoft/lalsuite>`__
+* `astropy <https://github.com/astropy/astropy>`__
+
+**Plotting**
+
+* `corner <https://github.com/dfm/corner.py>`__ for generating corner plot
+* `matplotlib <https://github.com/matplotlib/matplotlib>`__ for general plotting routines
+
+
 .. |pipeline status| image:: https://git.ligo.org/lscsoft/bilby/badges/master/pipeline.svg
    :target: https://git.ligo.org/lscsoft/bilby/commits/master
 .. |coverage report| image:: https://lscsoft.docs.ligo.org/bilby/coverage_badge.svg

bilby/core/prior.py

@@ -245,10 +245,28 @@ class PriorDict(OrderedDict):
         """
         return [self[key].rescale(sample) for key, sample in zip(keys, theta)]
 
-    def test_redundancy(self, key):
+    def test_redundancy(self, key, disable_logging=False):
         """Empty redundancy test, should be overwritten in subclasses"""
         return False
 
+    def test_has_redundant_keys(self):
+        """
+        Test whether there are redundant keys in self.
+
+        Return
+        ------
+        bool: Whether there are redundancies or not
+        """
+        redundant = False
+        for key in self:
+            temp = self.copy()
+            del temp[key]
+            if temp.test_redundancy(key, disable_logging=True):
+                logger.warning('{} is a redundant key in this {}.'
+                               .format(key, self.__class__.__name__))
+                redundant = True
+        return redundant
+
     def copy(self):
         """
         We have to overwrite the copy method as it fails due to the presence of

bilby/core/result.py

@@ -8,6 +8,7 @@ import numpy as np
 import deepdish
 import pandas as pd
 import corner
+import json
 import scipy.stats
 import matplotlib
 import matplotlib.pyplot as plt
@@ -19,7 +20,7 @@ from .utils import (logger, infer_parameters_from_function,
 from .prior import Prior, PriorDict, DeltaFunction
 
 
-def result_file_name(outdir, label):
+def result_file_name(outdir, label, extension='json'):
     """ Returns the standard filename used for a result file
 
     Parameters
@@ -28,43 +29,55 @@ def result_file_name(outdir, label):
         Name of the output directory
     label: str
         Naming scheme of the output file
+    extension: str, optional
+        Whether to save as `hdf5` or `json`
 
     Returns
     -------
     str: File name of the output file
     """
-    return '{}/{}_result.h5'.format(outdir, label)
+    if extension in ['json', 'hdf5']:
+        return '{}/{}_result.{}'.format(outdir, label, extension)
+    else:
+        raise ValueError("Extension type {} not understood".format(extension))
+
+
+def _determine_file_name(filename, outdir, label, extension):
+    """ Helper method to determine the filename """
+    if filename is not None:
+        return filename
+    else:
+        if (outdir is None) and (label is None):
+            raise ValueError("No information given to load file")
+        else:
+            return result_file_name(outdir, label, extension)
 
 
-def read_in_result(filename=None, outdir=None, label=None):
-    """ Read in a saved .h5 data file
+def read_in_result(filename=None, outdir=None, label=None, extension='json'):
+    """ Reads in a stored bilby result object
 
     Parameters
     ----------
     filename: str
-        If given, try to load from this filename
-    outdir, label: str
-        If given, use the default naming convention for saved results file
-
-    Returns
-    -------
-    result: bilby.core.result.Result
-
-    Raises
-    -------
-    ValueError: If no filename is given and either outdir or label is None
-                If no bilby.core.result.Result is found in the path
+        Path to the file to be read (alternative to giving the outdir and label)
+    outdir, label, extension: str
+        Name of the output directory, label and extension used for the default
+        naming scheme.
 
     """
-    if filename is None:
-        if (outdir is None) and (label is None):
-            raise ValueError("No information given to load file")
-        else:
-            filename = result_file_name(outdir, label)
-    if os.path.isfile(filename):
-        return Result(**deepdish.io.load(filename))
+    filename = _determine_file_name(filename, outdir, label, extension)
+
+    # Get the actual extension (may differ from the default extension if the filename is given)
+    extension = os.path.splitext(filename)[1].lstrip('.')
+    if 'json' in extension:
+        result = Result.from_json(filename=filename)
+    elif ('hdf5' in extension) or ('h5' in extension):
+        result = Result.from_hdf5(filename=filename)
+    elif extension is None:
+        raise ValueError("No filetype extension provided")
     else:
-        raise IOError("No result '{}' found".format(filename))
+        raise ValueError("Filetype {} not understood".format(extension))
+    return result
 
 
 class Result(object):
@@ -156,6 +169,81 @@ class Result(object):
         self.prior_values = None
         self._kde = None
 
+    @classmethod
+    def from_hdf5(cls, filename=None, outdir=None, label=None):
+        """ Read in a saved .h5 data file
+
+        Parameters
+        ----------
+        filename: str
+            If given, try to load from this filename
+        outdir, label: str
+            If given, use the default naming convention for saved results file
+
+        Returns
+        -------
+        result: bilby.core.result.Result
+
+        Raises
+        -------
+        ValueError: If no filename is given and either outdir or label is None
+                    If no bilby.core.result.Result is found in the path
+
+        """
+        filename = _determine_file_name(filename, outdir, label, 'hdf5')
+
+        if os.path.isfile(filename):
+            dictionary = deepdish.io.load(filename)
+            # Some versions of deepdish/pytables return the dictionanary as
+            # a dictionary with a key 'data'
+            if len(dictionary) == 1 and 'data' in dictionary:
+                dictionary = dictionary['data']
+            try:
+                return cls(**dictionary)
+            except TypeError as e:
+                raise IOError("Unable to load dictionary, error={}".format(e))
+        else:
+            raise IOError("No result '{}' found".format(filename))
+
+    @classmethod
+    def from_json(cls, filename=None, outdir=None, label=None):
+        """ Read in a saved .json data file
+
+        Parameters
+        ----------
+        filename: str
+            If given, try to load from this filename
+        outdir, label: str
+            If given, use the default naming convention for saved results file
+
+        Returns
+        -------
+        result: bilby.core.result.Result
+
+        Raises
+        -------
+        ValueError: If no filename is given and either outdir or label is None
+                    If no bilby.core.result.Result is found in the path
+
+        """
+        filename = _determine_file_name(filename, outdir, label, 'json')
+
+        if os.path.isfile(filename):
+            with open(filename, 'r') as file:
+                dictionary = json.load(file, object_hook=decode_bilby_json_result)
+            for key in dictionary.keys():
+                # Convert the loaded priors to bilby prior type
+                if key == 'priors':
+                    for param in dictionary[key].keys():
+                        dictionary[key][param] = str(dictionary[key][param])
+                    dictionary[key] = PriorDict(dictionary[key])
+            try:
+                return cls(**dictionary)
+            except TypeError as e:
+                raise IOError("Unable to load dictionary, error={}".format(e))
+        else:
+            raise IOError("No result '{}' found".format(filename))
+
     def __str__(self):
         """Print a summary """
         if getattr(self, 'posterior', None) is not None:
@@ -290,9 +378,9 @@ class Result(object):
                 pass
         return dictionary
 
-    def save_to_file(self, overwrite=False, outdir=None):
+    def save_to_file(self, overwrite=False, outdir=None, extension='json'):
         """
-        Writes the Result to a deepdish h5 file
+        Writes the Result to a json or deepdish h5 file
 
         Parameters
         ----------
@@ -301,9 +389,11 @@ class Result(object):
             default=False
         outdir: str, optional
             Path to the outdir. Default is the one stored in the result object.
+        extension: str, optional {json, hdf5}
+            Determines the method to use to store the data
         """
         outdir = self._safe_outdir_creation(outdir, self.save_to_file)
-        file_name = result_file_name(outdir, self.label)
+        file_name = result_file_name(outdir, self.label, extension)
 
         if os.path.isfile(file_name):
             if overwrite:
@@ -322,14 +412,20 @@ class Result(object):
         if dictionary.get('priors', False):
             dictionary['priors'] = {key: str(self.priors[key]) for key in self.priors}
 
-        # Convert callable sampler_kwargs to strings to avoid pickling issues
+        # Convert callable sampler_kwargs to strings
         if dictionary.get('sampler_kwargs', None) is not None:
             for key in dictionary['sampler_kwargs']:
                 if hasattr(dictionary['sampler_kwargs'][key], '__call__'):
                     dictionary['sampler_kwargs'][key] = str(dictionary['sampler_kwargs'])
 
         try:
-            deepdish.io.save(file_name, dictionary)
+            if extension == 'json':
+                with open(file_name, 'w') as file:
+                    json.dump(dictionary, file, indent=2, cls=BilbyResultJsonEncoder)
+            elif extension == 'hdf5':
+                deepdish.io.save(file_name, dictionary)
+            else:
+                raise ValueError("Extension type {} not understood".format(extension))
         except Exception as e:
             logger.error("\n\n Saving the data has failed with the "
                          "following message:\n {} \n\n".format(e))
@@ -1026,6 +1122,27 @@ class Result(object):
         return outdir
 
 
+class BilbyResultJsonEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return {'__array__': True, 'content': obj.tolist()}
+        if isinstance(obj, complex):
+            return {'__complex__': True, 'real': obj.real, 'imag': obj.imag}
+        if isinstance(obj, pd.core.frame.DataFrame):
+            return {'__dataframe__': True, 'content': obj.to_dict(orient='list')}
+        return json.JSONEncoder.default(self, obj)
+
+
+def decode_bilby_json_result(dct):
+    if dct.get("__array__", False):
+        return np.asarray(dct["content"])
+    if dct.get("__complex__", False):
+        return complex(dct["real"], dct["imag"])
+    if dct.get("__dataframe__", False):
+        return pd.DataFrame(dct['content'])
+    return dct
+
+
 def plot_multiple(results, filename=None, labels=None, colours=None,
                   save=True, evidences=False, **kwargs):
     """ Generate a corner plot overlaying two sets of results

bilby/core/sampler/__init__.py

@@ -11,20 +11,21 @@ from .cpnest import Cpnest
 from .dynesty import Dynesty
 from .emcee import Emcee
 from .nestle import Nestle
+from .polychord import PyPolyChord
 from .ptemcee import Ptemcee
 from .ptmcmc import PTMCMCSampler
 from .pymc3 import Pymc3
 from .pymultinest import Pymultinest
 
-implemented_samplers = {
+IMPLEMENTED_SAMPLERS = {
     'cpnest': Cpnest, 'dynesty': Dynesty, 'emcee': Emcee, 'nestle': Nestle,
     'ptemcee': Ptemcee,'ptmcmcsampler' : PTMCMCSampler,
-    'pymc3': Pymc3, 'pymultinest': Pymultinest }
+    'pymc3': Pymc3, 'pymultinest': Pymultinest, 'pypolychord': PyPolyChord }
 
 if command_line_args.sampler_help:
     sampler = command_line_args.sampler_help
-    if sampler in implemented_samplers:
-        sampler_class = implemented_samplers[sampler]
+    if sampler in IMPLEMENTED_SAMPLERS:
+        sampler_class = IMPLEMENTED_SAMPLERS[sampler]
         print('Help for sampler "{}":'.format(sampler))
         print(sampler_class.__doc__)
     else:
@@ -33,7 +34,7 @@ if command_line_args.sampler_help:
                   'the name of the sampler')
         else:
             print('Requested sampler {} not implemented'.format(sampler))
-        print('Available samplers = {}'.format(implemented_samplers))
+        print('Available samplers = {}'.format(IMPLEMENTED_SAMPLERS))
 
     sys.exit()
 
@@ -84,6 +85,7 @@ def run_sampler(likelihood, priors=None, label='label', outdir='outdir',
         overwritten.
     save: bool
         If true, save the priors and results to disk.
+        If hdf5, save as an hdf5 file instead of json.
     result_class: bilby.core.result.Result, or child of
         The result class to use. By default, `bilby.core.result.Result` is used,
         but objects which inherit from this class can be given providing
@@ -106,7 +108,7 @@ def run_sampler(likelihood, priors=None, label='label', outdir='outdir',
     if command_line_args.clean:
         kwargs['resume'] = False
 
-    from . import implemented_samplers
+    from . import IMPLEMENTED_SAMPLERS
 
     if priors is None:
         priors = dict()
@@ -128,15 +130,15 @@ def run_sampler(likelihood, priors=None, label='label', outdir='outdir',
     if isinstance(sampler, Sampler):
         pass
     elif isinstance(sampler, str):
-        if sampler.lower() in implemented_samplers:
-            sampler_class = implemented_samplers[sampler.lower()]
+        if sampler.lower() in IMPLEMENTED_SAMPLERS:
+            sampler_class = IMPLEMENTED_SAMPLERS[sampler.lower()]
             sampler = sampler_class(
                 likelihood, priors=priors, outdir=outdir, label=label,
                 injection_parameters=injection_parameters, meta_data=meta_data,
                 use_ratio=use_ratio, plot=plot, result_class=result_class,
                 **kwargs)
         else:
-            print(implemented_samplers)
+            print(IMPLEMENTED_SAMPLERS)
             raise ValueError(
                 "Sampler {} not yet implemented".format(sampler))
     elif inspect.isclass(sampler):
@@ -148,7 +150,7 @@ def run_sampler(likelihood, priors=None, label='label', outdir='outdir',
     else:
         raise ValueError(
             "Provided sampler should be a Sampler object or name of a known "
-            "sampler: {}.".format(', '.join(implemented_samplers.keys())))
+            "sampler: {}.".format(', '.join(IMPLEMENTED_SAMPLERS.keys())))
 
     if sampler.cached_result:
         logger.warning("Using cached result")
@@ -180,9 +182,12 @@ def run_sampler(likelihood, priors=None, label='label', outdir='outdir',
             result.injection_parameters = conversion_function(
                 result.injection_parameters)
 
-    result.samples_to_posterior(likelihood=likelihood, priors=priors,
+    result.samples_to_posterior(likelihood=likelihood, priors=result.priors,
                                 conversion_function=conversion_function)
-    if save:
+    if save == 'hdf5':
+        result.save_to_file(extension='hdf5')
+        logger.info("Results saved to {}/".format(outdir))
+    elif save:
         result.save_to_file()
         logger.info("Results saved to {}/".format(outdir))
     if plot:

bilby/core/sampler/base_sampler.py

@@ -241,6 +241,10 @@ class Sampler(object):
             Likelihood can't be evaluated.
 
         """
+
+        if self.priors.test_has_redundant_keys():
+            raise IllegalSamplingSetError("Your sampling set contains redundant parameters.")
+
         self._check_if_priors_can_be_sampled()
         try:
             t1 = datetime.datetime.now()
@@ -502,3 +506,7 @@ class SamplerError(Error):
 
 class SamplerNotInstalledError(SamplerError):
     """ Base class for Error raised by not installed samplers """
+
+
+class IllegalSamplingSetError(Error):
+    """ Class for illegal sets of sampling parameters """

bilby/core/sampler/dynesty.py

@@ -2,10 +2,11 @@ from __future__ import absolute_import
 
 import os
 import sys
+import pickle
+import signal
 
 import numpy as np
 from pandas import DataFrame
-from deepdish.io import load, save
 
 from ..utils import logger, check_directory_exists_and_if_not_mkdir
 from .base_sampler import Sampler, NestedSampler
@@ -103,6 +104,11 @@ class Dynesty(NestedSampler):
             n_check_point_rnd = int(float("{:1.0g}".format(n_check_point_raw)))
             self.n_check_point = n_check_point_rnd
 
+        self.resume_file = '{}/{}_resume.pickle'.format(self.outdir, self.label)
+
+        signal.signal(signal.SIGTERM, self.write_current_state_and_exit)
+        signal.signal(signal.SIGINT, self.write_current_state_and_exit)
+
     @property
     def sampler_function_kwargs(self):
         keys = ['dlogz', 'print_progress', 'print_func', 'maxiter',
@@ -236,15 +242,15 @@ class Dynesty(NestedSampler):
 
     def _remove_checkpoint(self):
         """Remove checkpointed state"""
-        if os.path.isfile('{}/{}_resume.h5'.format(self.outdir, self.label)):
-            os.remove('{}/{}_resume.h5'.format(self.outdir, self.label))
+        if os.path.isfile(self.resume_file):
+            os.remove(self.resume_file)
 
     def read_saved_state(self, continuing=False):
         """
         Read a saved state of the sampler to disk.
 
         The required information to reconstruct the state of the run is read
-        from an hdf5 file.
+        from a pickle file.