diff --git a/bilby/core/prior/analytical.py b/bilby/core/prior/analytical.py
index 03fb71e15d19c209e43293860002d254e6568ff8..4de79a382021dbc679e3eb81a7035b16f91fcb28 100644
--- a/bilby/core/prior/analytical.py
+++ b/bilby/core/prior/analytical.py
@@ -25,7 +25,7 @@ class DeltaFunction(Prior):
 
         """
         super(DeltaFunction, self).__init__(name=name, latex_label=latex_label, unit=unit,
-                                            minimum=peak, maximum=peak)
+                                            minimum=peak, maximum=peak, check_range_nonzero=False)
         self.peak = peak
         self._is_fixed = True
 
diff --git a/bilby/core/prior/base.py b/bilby/core/prior/base.py
index 820b4965d3cebe7a3f9da9eb4bd683cc51bee97d..29efe925d1db9c9dde574c45cd280d4ab2e746b3 100644
--- a/bilby/core/prior/base.py
+++ b/bilby/core/prior/base.py
@@ -15,7 +15,7 @@ class Prior(object):
     _default_latex_labels = {}
 
     def __init__(self, name=None, latex_label=None, unit=None, minimum=-np.inf,
-                 maximum=np.inf, boundary=None):
+                 maximum=np.inf, check_range_nonzero=True, boundary=None):
         """ Implements a Prior object
 
         Parameters
@@ -30,15 +30,24 @@ class Prior(object):
             Minimum of the domain, default=-np.inf
         maximum: float, optional
             Maximum of the domain, default=np.inf
+        check_range_nonzero: boolean, optional
+            If True, checks that the prior range is non-zero
         boundary: str, optional
             The boundary condition of the prior, can be 'periodic', 'reflective'
             Currently implemented in cpnest, dynesty and pymultinest.
         """
+        if check_range_nonzero and maximum <= minimum:
+            raise ValueError(
+                "maximum {} <= minimum {} for {} prior on {}".format(
+                    maximum, minimum, type(self).__name__, name
+                )
+            )
         self.name = name
         self.latex_label = latex_label
         self.unit = unit
         self.minimum = minimum
         self.maximum = maximum
+        self.check_range_nonzero = check_range_nonzero
         self.least_recently_sampled = None
         self.boundary = boundary
         self._is_fixed = False
diff --git a/test/prior_test.py b/test/prior_test.py
index 8b2b9efe9c1f73e67c0ef8dab5256b777bfb35b2..af0fbb909a654f29b682f7859ffd3368b7974b4b 100644
--- a/test/prior_test.py
+++ b/test/prior_test.py
@@ -42,9 +42,9 @@ class TestPriorInstantiationWithoutOptionalPriors(unittest.TestCase):
         arguments.
         """
         self.prior = bilby.core.prior.Prior(name='test_name', latex_label='test_label', minimum=0, maximum=1,
-                                            boundary=None)
+                                            check_range_nonzero=True, boundary=None)
         expected_string = "Prior(name='test_name', latex_label='test_label', unit=None, minimum=0, maximum=1, " \
-                          "boundary=None)"
+                          "check_range_nonzero=True, boundary=None)"
         self.assertTrue(sorted(expected_string) == sorted(self.prior.__repr__()))
 
     def test_base_prob(self):