diff --git a/octodns/provider/base.py b/octodns/provider/base.py
index 2fd4349..d2561ba 100644
--- a/octodns/provider/base.py
+++ b/octodns/provider/base.py
@@ -56,19 +56,19 @@ class Plan(object):
             delete_pcent = self.change_counts['Delete'] / existing_record_count
 
             if update_pcent > self.MAX_SAFE_UPDATE_PCENT:
-                raise UnsafePlan('Too many updates, %s is over %s percent'
-                                 '(%s/%s)',
-                                 update_pcent,
-                                 self.MAX_SAFE_UPDATE_PCENT * 100,
-                                 self.change_counts['Update'],
-                                 existing_record_count)
+                raise UnsafePlan('Too many updates, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     update_pcent,
+                                     self.MAX_SAFE_UPDATE_PCENT * 100,
+                                     self.change_counts['Update'],
+                                     existing_record_count))
             if delete_pcent > self.MAX_SAFE_DELETE_PCENT:
-                raise UnsafePlan('Too many deletes, %s is over %s percent'
-                                 '(%s/%s)',
-                                 delete_pcent,
-                                 self.MAX_SAFE_DELETE_PCENT * 100,
-                                 self.change_counts['Delete'],
-                                 existing_record_count)
+                raise UnsafePlan('Too many deletes, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     delete_pcent,
+                                     self.MAX_SAFE_DELETE_PCENT * 100,
+                                     self.change_counts['Delete'],
+                                     existing_record_count))
 
     def __repr__(self):
         return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py
index bd134bc..e44adc0 100644
--- a/tests/test_octodns_provider_base.py
+++ b/tests/test_octodns_provider_base.py
@@ -214,9 +214,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_UPDATE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many updates' in ctx.exception.message)
+
     def test_safe_updates_min_existing_pcent(self):
         # MAX_SAFE_UPDATE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist
@@ -260,9 +262,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_DELETE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many deletes' in ctx.exception.message)
+
     def test_safe_deletes_min_existing_pcent(self):
         # MAX_SAFE_DELETE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist