Update pp plots

Put confidence bands on the pp plots. This is pretty untested though-- sorry!

bilby/core/result.py

@@ -1122,7 +1122,7 @@ def plot_multiple(results, filename=None, labels=None, colours=None,
     return fig
 
 
-def make_pp_plot(results, filename=None, save=True, **kwargs):
+def make_pp_plot(results, filename=None, save=True, confidence_interval=0.9, **kwargs):
     """
     Make a P-P plot for a set of runs with injected signals.
 
@@ -1134,6 +1134,8 @@ def make_pp_plot(results, filename=None, save=True, **kwargs):
         The name of the file to save, the default is "outdir/pp.png"
     save: bool, optional
         Whether to save the file, default=True
+    confidence_interval: float, optional
+        The confidence interval to be plotted, defaulting to 0.9 (90%)
     kwargs:
         Additional kwargs to pass to matplotlib.pyplot.plot
 
@@ -1149,11 +1151,22 @@ def make_pp_plot(results, filename=None, save=True, **kwargs):
             result.get_all_injection_credible_levels(), ignore_index=True)
     n_parameters = len(credible_levels.keys())
     x_values = np.linspace(0, 1, 101)
+
+    # Putting in the confidence bands
+    N = len(credible_levels)
+    edge_of_bound = (1.-confidence_interval)/2.
+    lower = scipy.stats.binom.ppf(1-edge_of_bound, N, x_values)/N
+    upper = scipy.stats.binom.ppf(edge_of_bound ,N, x_values)/N
+    # The binomial point percent function doesn't always return 0 @ 0, so set those bounds explicitly to be sure
+    lower[0] = 0
+    upper[0] = 0
+
     for key in credible_levels:
         plt.plot(x_values, [sum(credible_levels[key].values < xx) /
                             len(credible_levels) for xx in x_values],
                  color='k', alpha=min([1, 4 / n_parameters]), **kwargs)
     plt.plot([0, 1], [0, 1], linestyle='--', color='r')
+    plt.fill_between(x_values, lower, upper, alpha=0.4)
     plt.xlim(0, 1)
     plt.ylim(0, 1)
     plt.tight_layout()