Add some post processing tools

This MR modifies how the marginal histograms are generated now:

• everything in the posterior is plotted by default.
• the pdf and cdf are plotted.

It also adds methods to the result to get the credible level at which injections are found and a function to make a pp-plot from many injections.

bilby/core/result.py

+from __future__ import division
+
 import os
 from distutils.version import LooseVersion
 import numpy as np
@@ -9,7 +11,8 @@ import matplotlib.pyplot as plt
 from collections import OrderedDict, namedtuple
 
 from . import utils
-from .utils import logger, infer_parameters_from_function
+from .utils import (logger, infer_parameters_from_function,
+                    check_directory_exists_and_if_not_mkdir)
 from .prior import PriorDict, DeltaFunction
 
 
@@ -136,6 +139,7 @@ class Result(object):
         self.log_bayes_factor = log_bayes_factor
         self.log_likelihood_evaluations = log_likelihood_evaluations
         self.sampling_time = sampling_time
+        self.max_autocorrelation_time = max_autocorrelation_time
 
     def __str__(self):
         """Print a summary """
@@ -327,7 +331,7 @@ class Result(object):
             elif k in self.parameter_labels:
                 latex_labels.append(k)
             else:
-                logger.info(
+                logger.debug(
                     'key {} not a parameter label or latex label'.format(k))
                 latex_labels.append(' '.join(k.split('_')))
         return latex_labels
@@ -433,40 +437,45 @@ class Result(object):
 
         Returns
         -------
-        figures: dictionary
-            A dictionary of the matplotlib figures
+        # figures: dictionary
+        #     A dictionary of the matplotlib figures
 
         """
-
         if isinstance(parameters, dict):
             plot_parameter_keys = list(parameters.keys())
             truths = list(parameters.values())
         elif parameters is None:
-            plot_parameter_keys = self.search_parameter_keys
-            truths = None
+            plot_parameter_keys = self.injection_parameters.keys()
+            truths = [self.injection_parameters.get(key, None) for key
+                      in plot_parameter_keys]
         else:
             plot_parameter_keys = list(parameters)
-            truths = None
+            truths = [self.injection_parameters.get(key, None) for key
+                      in plot_parameter_keys]
 
         labels = self.get_latex_labels_from_parameter_keys(plot_parameter_keys)
         if file_base_name is None:
-            file_base_name = '{}/{}_'.format(self.outdir, self.label)
+            file_base_name = '{}/{}_1d/'.format(self.outdir, self.label)
+            check_directory_exists_and_if_not_mkdir(file_base_name)
 
         if priors is True:
             priors = getattr(self, 'priors', False)
-        elif isinstance(priors, (dict)) or priors in [False, None]:
+        elif isinstance(priors, dict) or priors in [False, None]:
             pass
         else:
             raise ValueError('Input priors={} not understood'.format(priors))
 
-        figures = dict()
+        # figures = dict()
         for i, key in enumerate(plot_parameter_keys):
+            if not isinstance(self.posterior[key].values[0], float):
+                continue
+            logger.info('Plotting {} marginal distribution'.format(key))
             fig, ax = plt.subplots()
             ax.hist(self.posterior[key].values, bins=bins, density=True,
                     histtype='step')
             ax.set_xlabel(labels[i], fontsize=label_fontsize)
-            if truths is not None:
-                ax.axvline(truths[i], ls='--', color='orange')
+            if truths[i] is not None:
+                ax.axvline(truths[i], ls='-', color='orange')
 
             summary = self.get_one_dimensional_median_and_error_bar(
                 key, quantiles=quantiles)
@@ -475,15 +484,32 @@ class Result(object):
             if titles:
                 ax.set_title(summary.string, fontsize=title_fontsize)
 
-            if isinstance(priors, dict):
+            if isinstance(priors, dict) and key in priors:
                 theta = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], 300)
                 ax.plot(theta, priors[key].prob(theta), color='C2')
 
             fig.tight_layout()
-            fig.savefig(file_base_name + key)
-            figures[key] = fig
+            fig.savefig(file_base_name + key + '_pdf', dpi=dpi)
+            plt.close(fig)
+
+            fig, ax = plt.subplots()
+            ax.hist(self.posterior[key].values, bins=bins, density=True,
+                    histtype='step', cumulative=True)
+            ax.set_xlabel(labels[i], fontsize=label_fontsize)
+            if truths[i] is not None:
+                ax.axvline(truths[i], ls='-', color='orange')
 
-        return figures
+            ax.axvline(summary.median - summary.minus, ls='--', color='C0')
+            ax.axvline(summary.median + summary.plus, ls='--', color='C0')
+            if titles:
+                ax.set_title(summary.string, fontsize=title_fontsize)
+
+            fig.tight_layout()
+            fig.savefig(file_base_name + key + '_cdf', dpi=dpi)
+            plt.close(fig)
+            # figures[key] = fig
+
+        # return figures
 
     def plot_corner(self, parameters=None, priors=None, titles=True, save=True,
                     filename=None, dpi=300, **kwargs):
@@ -777,6 +803,43 @@ class Result(object):
                     self.prior_values[key]\
                         = priors[key].prob(self.posterior[key].values)
 
+    def get_all_injection_credible_levels(self):
+        """
+        Get credible levels for all parameters in self.injection_parameters
+
+        Returns
+        -------
+        credible_levels: dict
+            The credible levels at which the injected parameters are found.
+        """
+        credible_levels = {key: self.get_injection_credible_level(key)
+                           for key in self.search_parameter_keys
+                           if isinstance(self.injection_parameters[key], float)}
+        return credible_levels
+
+    def get_injection_credible_level(self, parameter):
+        """
+        Get the credible level of the injected parameter
+
+        Calculated as CDF(injection value)
+
+        Parameters
+        ----------
+        parameter: str
+            Parameter to get credible level for
+        Returns
+        -------
+        float: credible level
+        """
+        if parameter in self.posterior and\
+                parameter in self.injection_parameters:
+            credible_level =\
+                sum(self.posterior[parameter].values <
+                    self.injection_parameters[parameter]) / len(self.posterior)
+            return credible_level
+        else:
+            return np.nan
+
     def _check_attribute_match_to_other_object(self, name, other_object):
         """ Check attribute name exists in other_object and is the same
 
@@ -890,3 +953,45 @@ def plot_multiple(results, filename=None, labels=None, colours=None,
     if save:
         fig.savefig(filename)
     return fig
+
+
+def make_pp_plot(results, filename=None, save=True, **kwargs):
+    """
+    Make a P-P plot for a set of runs with injected signals.
+
+    Parameters
+    ----------
+    results: list
+        A list of Result objects, each of these should have injected_parameters
+    filename: str, optional
+        The name of the file to save, the default is "outdir/pp.png"
+    save: bool, optional
+        Whether to save the file, default=True
+    kwargs:
+        Additional kwargs to pass to matplotlib.pyplot.plot
+
+    Returns
+    -------
+    fig:
+        Matplotlib figure
+    """
+    fig = plt.figure()
+    credible_levels = pd.DataFrame()
+    for result in results:
+        credible_levels = credible_levels.append(
+            result.get_all_injection_credible_levels(), ignore_index=True)
+    n_parameters = len(credible_levels.keys())
+    x_values = np.linspace(0, 1, 101)
+    for key in credible_levels:
+        plt.plot(x_values, [sum(credible_levels[key].values < xx) /
+                            len(credible_levels) for xx in x_values],
+                 color='k', alpha=min([1, 4 / n_parameters]), **kwargs)
+    plt.plot([0, 1], [0, 1], linestyle='--', color='r')
+    plt.xlim(0, 1)
+    plt.ylim(0, 1)
+    plt.tight_layout()
+    if save:
+        if filename is None:
+            filename = 'outdir/pp.png'
+        plt.savefig(filename)
+    return fig