From 90cc9576f540e43de4b7ac5d7bfee02a87ea5f58 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:46:29 -0400 Subject: [PATCH 1/3] Increase Cloudflare page size Increase Cloudflare page size to reduce request count `GET zones` has a MAX of 50 and a default of 20 https://api.cloudflare.com/#zone-list-zones `GET zones/:zone_identifier/dns_records` has a MAX of 100 and a default of 20 https://api.cloudflare.com/#dns-records-for-a-zone-list-dns-records --- octodns/provider/cloudflare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 698fbee..96febf4 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -142,7 +142,7 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page}) + params={'page': page, 'per_page': 50}) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -251,7 +251,7 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page}) + resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From b80d1575e6d5268e3ee69c14adcc8e2ad9dcfa88 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:56:31 -0400 Subject: [PATCH 2/3] Update tests with new per_page params --- octodns/provider/cloudflare.py | 3 ++- tests/test_octodns_provider_cloudflare.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96febf4..96c9f5e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -251,7 +251,8 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) + resp = self._try_request('GET', path, params={'page': page, + 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 08608ea..735d95c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase): # get the list of zones, create a zone, add some records, update # something, and delete something provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase): # Get zones, create zone, create a record, delete a record provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase): provider._request.side_effect = [result] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # One retry required provider._zones = None @@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # Two retries required provider._zones = None @@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # # Exhaust our retries provider._zones = None From 84048dbde9df4de639141d32ca4722c538b2b92b Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Mon, 22 Jun 2020 17:27:41 -0400 Subject: [PATCH 3/3] Cloudflare: Make page size configurable --- octodns/provider/cloudflare.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96c9f5e..db937e5 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider): retry_count: 4 # Optional. Default: 300. Number of seconds to wait before retrying. retry_period: 300 + # Optional. Default: 50. Number of zones per page. + zones_per_page: 50 + # Optional. Default: 100. Number of dns records per page. + records_per_page: 100 Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed via the YAML provider like so: @@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider): TIMEOUT = 15 def __init__(self, id, email=None, token=None, cdn=False, retry_count=4, - retry_period=300, *args, **kwargs): + retry_period=300, zones_per_page=50, records_per_page=100, + *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) @@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider): self.cdn = cdn self.retry_count = retry_count self.retry_period = retry_period + self.zones_per_page = zones_per_page + self.records_per_page = records_per_page self._sess = sess self._zones = None @@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page, 'per_page': 50}) + params={ + 'page': page, + 'per_page': self.zones_per_page + }) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -252,7 +262,7 @@ class CloudflareProvider(BaseProvider): page = 1 while page: resp = self._try_request('GET', path, params={'page': page, - 'per_page': 100}) + 'per_page': self.records_per_page}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: