1
0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Files
github-octodns/octodns/provider/cloudflare.py

718 lines
26 KiB
Python

#
#
#
from __future__ import absolute_import, division, print_function, \
unicode_literals
from collections import defaultdict
from copy import deepcopy
from logging import getLogger
from requests import Session
from time import sleep
from ..record import Record, Update
from .base import BaseProvider
class CloudflareError(Exception):
def __init__(self, data):
try:
message = data['errors'][0]['message']
except (IndexError, KeyError, TypeError):
message = 'Cloudflare error'
super(CloudflareError, self).__init__(message)
class CloudflareAuthenticationError(CloudflareError):
def __init__(self, data):
CloudflareError.__init__(self, data)
class CloudflareRateLimitError(CloudflareError):
def __init__(self, data):
CloudflareError.__init__(self, data)
_PROXIABLE_RECORD_TYPES = {'A', 'AAAA', 'ALIAS', 'CNAME'}
class CloudflareProvider(BaseProvider):
'''
Cloudflare DNS provider
cloudflare:
class: octodns.provider.cloudflare.CloudflareProvider
# The api key (required)
# Your Cloudflare account email address (required)
email: dns-manager@example.com (optional if using token)
token: foo
# Import CDN enabled records as CNAME to {}.cdn.cloudflare.net. Records
# ending at .cdn.cloudflare.net. will be ignored when this provider is
# not used as the source and the cdn option is enabled.
#
# See: https://support.cloudflare.com/hc/en-us/articles/115000830351
cdn: false
# Optional. Default: 4. Number of times to retry if a 429 response
# is received.
retry_count: 4
# Optional. Default: 300. Number of seconds to wait before retrying.
retry_period: 300
# Optional. Default: 50. Number of zones per page.
zones_per_page: 50
# Optional. Default: 100. Number of dns records per page.
records_per_page: 100
Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed
via the YAML provider like so:
name:
octodns:
cloudflare:
proxied: true
ttl: 120
type: A
value: 1.2.3.4
'''
SUPPORTS_GEO = False
SUPPORTS_DYNAMIC = False
SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS',
'PTR', 'SRV', 'SPF', 'TXT'))
MIN_TTL = 120
TIMEOUT = 15
def __init__(self, id, email=None, token=None, cdn=False, retry_count=4,
retry_period=300, zones_per_page=50, records_per_page=100,
*args, **kwargs):
self.log = getLogger('CloudflareProvider[{}]'.format(id))
self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id,
email, cdn)
super(CloudflareProvider, self).__init__(id, *args, **kwargs)
sess = Session()
if email and token:
sess.headers.update({
'X-Auth-Email': email,
'X-Auth-Key': token,
})
else:
# https://api.cloudflare.com/#getting-started-requests
# https://tools.ietf.org/html/rfc6750#section-2.1
sess.headers.update({
'Authorization': 'Bearer {}'.format(token),
})
self.cdn = cdn
self.retry_count = retry_count
self.retry_period = retry_period
self.zones_per_page = zones_per_page
self.records_per_page = records_per_page
self._sess = sess
self._zones = None
self._zone_records = {}
def _try_request(self, *args, **kwargs):
tries = self.retry_count
while True: # We'll raise to break after our tries expire
try:
return self._request(*args, **kwargs)
except CloudflareRateLimitError:
if tries <= 1:
raise
tries -= 1
self.log.warn('rate limit encountered, pausing '
'for %ds and trying again, %d remaining',
self.retry_period, tries)
sleep(self.retry_period)
def _request(self, method, path, params=None, data=None):
self.log.debug('_request: method=%s, path=%s', method, path)
url = 'https://api.cloudflare.com/client/v4{}'.format(path)
resp = self._sess.request(method, url, params=params, json=data,
timeout=self.TIMEOUT)
self.log.debug('_request: status=%d', resp.status_code)
if resp.status_code == 400:
self.log.debug('_request: data=%s', data)
raise CloudflareError(resp.json())
if resp.status_code == 403:
raise CloudflareAuthenticationError(resp.json())
if resp.status_code == 429:
raise CloudflareRateLimitError(resp.json())
resp.raise_for_status()
return resp.json()
def _change_keyer(self, change):
key = change.__class__.__name__
order = {'Delete': 0, 'Create': 1, 'Update': 2}
return order[key]
@property
def zones(self):
if self._zones is None:
page = 1
zones = []
while page:
resp = self._try_request('GET', '/zones',
params={
'page': page,
'per_page': self.zones_per_page
})
zones += resp['result']
info = resp['result_info']
if info['count'] > 0 and info['count'] == info['per_page']:
page += 1
else:
page = None
self._zones = {'{}.'.format(z['name']): z['id'] for z in zones}
return self._zones
def _data_for_cdn(self, name, _type, records):
self.log.info('CDN rewrite for %s', records[0]['name'])
_type = "CNAME"
if name == "":
_type = "ALIAS"
return {
'ttl': records[0]['ttl'],
'type': _type,
'value': '{}.cdn.cloudflare.net.'.format(records[0]['name']),
}
def _data_for_multiple(self, _type, records):
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': [r['content'] for r in records],
}
_data_for_A = _data_for_multiple
_data_for_AAAA = _data_for_multiple
_data_for_SPF = _data_for_multiple
def _data_for_TXT(self, _type, records):
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': [r['content'].replace(';', '\\;') for r in records],
}
def _data_for_CAA(self, _type, records):
values = []
for r in records:
data = r['data']
values.append(data)
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': values,
}
def _data_for_CNAME(self, _type, records):
only = records[0]
return {
'ttl': only['ttl'],
'type': _type,
'value': '{}.'.format(only['content'])
}
_data_for_ALIAS = _data_for_CNAME
_data_for_PTR = _data_for_CNAME
def _data_for_LOC(self, _type, records):
values = []
for record in records:
r = record['data']
values.append({
'lat_degrees': int(r['lat_degrees']),
'lat_minutes': int(r['lat_minutes']),
'lat_seconds': float(r['lat_seconds']),
'lat_direction': r['lat_direction'],
'long_degrees': int(r['long_degrees']),
'long_minutes': int(r['long_minutes']),
'long_seconds': float(r['long_seconds']),
'long_direction': r['long_direction'],
'altitude': float(r['altitude']),
'size': float(r['size']),
'precision_horz': float(r['precision_horz']),
'precision_vert': float(r['precision_vert']),
})
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': values
}
def _data_for_MX(self, _type, records):
values = []
for r in records:
values.append({
'preference': r['priority'],
'exchange': '{}.'.format(r['content']),
})
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': values,
}
def _data_for_NS(self, _type, records):
return {
'ttl': records[0]['ttl'],
'type': _type,
'values': ['{}.'.format(r['content']) for r in records],
}
def _data_for_SRV(self, _type, records):
values = []
for r in records:
target = ('{}.'.format(r['data']['target'])
if r['data']['target'] != "." else ".")
values.append({
'priority': r['data']['priority'],
'weight': r['data']['weight'],
'port': r['data']['port'],
'target': target,
})
return {
'type': _type,
'ttl': records[0]['ttl'],
'values': values
}
def zone_records(self, zone):
if zone.name not in self._zone_records:
zone_id = self.zones.get(zone.name, False)
if not zone_id:
return []
records = []
path = '/zones/{}/dns_records'.format(zone_id)
page = 1
while page:
resp = self._try_request('GET', path, params={'page': page,
'per_page': self.records_per_page})
records += resp['result']
info = resp['result_info']
if info['count'] > 0 and info['count'] == info['per_page']:
page += 1
else:
page = None
self._zone_records[zone.name] = records
return self._zone_records[zone.name]
def _record_for(self, zone, name, _type, records, lenient):
# rewrite Cloudflare proxied records
if self.cdn and records[0]['proxied']:
data = self._data_for_cdn(name, _type, records)
else:
# Cloudflare supports ALIAS semantics with root CNAMEs
if _type == 'CNAME' and name == '':
_type = 'ALIAS'
data_for = getattr(self, '_data_for_{}'.format(_type))
data = data_for(_type, records)
record = Record.new(zone, name, data, source=self, lenient=lenient)
if _type in _PROXIABLE_RECORD_TYPES:
record._octodns['cloudflare'] = {
'proxied': records[0].get('proxied', False)
}
return record
def populate(self, zone, target=False, lenient=False):
self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
target, lenient)
exists = False
before = len(zone.records)
records = self.zone_records(zone)
if records:
exists = True
values = defaultdict(lambda: defaultdict(list))
for record in records:
name = zone.hostname_from_fqdn(record['name'])
_type = record['type']
if _type in self.SUPPORTS:
values[name][record['type']].append(record)
for name, types in values.items():
for _type, records in types.items():
record = self._record_for(zone, name, _type, records,
lenient)
# only one rewrite is needed for names where the proxy is
# enabled at multiple records with a different type but
# the same name
if (self.cdn and records[0]['proxied'] and
record in zone._records[name]):
self.log.info('CDN rewrite %s already in zone', name)
continue
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
return exists
def _include_change(self, change):
if isinstance(change, Update):
new = change.new.data
# Cloudflare manages TTL of proxied records, so we should exclude
# TTL from the comparison (to prevent false-positives).
if self._record_is_proxied(change.existing):
existing = deepcopy(change.existing.data)
existing.update({
'ttl': new['ttl']
})
else:
existing = change.existing.data
new['ttl'] = max(self.MIN_TTL, new['ttl'])
if new == existing:
return False
# If this is a record to enable Cloudflare CDN don't update as
# we don't know the original values.
if (change.record._type in ('ALIAS', 'CNAME') and
change.record.value.endswith('.cdn.cloudflare.net.')):
return False
return True
def _contents_for_multiple(self, record):
for value in record.values:
yield {'content': value}
_contents_for_A = _contents_for_multiple
_contents_for_AAAA = _contents_for_multiple
_contents_for_NS = _contents_for_multiple
_contents_for_SPF = _contents_for_multiple
def _contents_for_CAA(self, record):
for value in record.values:
yield {
'data': {
'flags': value.flags,
'tag': value.tag,
'value': value.value,
}
}
def _contents_for_TXT(self, record):
for value in record.values:
yield {'content': value.replace('\\;', ';')}
def _contents_for_CNAME(self, record):
yield {'content': record.value}
_contents_for_PTR = _contents_for_CNAME
def _contents_for_LOC(self, record):
for value in record.values:
yield {
'data': {
'lat_degrees': value.lat_degrees,
'lat_minutes': value.lat_minutes,
'lat_seconds': value.lat_seconds,
'lat_direction': value.lat_direction,
'long_degrees': value.long_degrees,
'long_minutes': value.long_minutes,
'long_seconds': value.long_seconds,
'long_direction': value.long_direction,
'altitude': value.altitude,
'size': value.size,
'precision_horz': value.precision_horz,
'precision_vert': value.precision_vert,
}
}
def _contents_for_MX(self, record):
for value in record.values:
yield {
'priority': value.preference,
'content': value.exchange
}
def _contents_for_SRV(self, record):
try:
service, proto, subdomain = record.name.split('.', 2)
# We have a SRV in a sub-zone
except ValueError:
# We have a SRV in the zone
service, proto = record.name.split('.', 1)
subdomain = None
name = record.zone.name
if subdomain:
name = subdomain
for value in record.values:
target = value.target[:-1] if value.target != "." else "."
yield {
'data': {
'service': service,
'proto': proto,
'name': name,
'priority': value.priority,
'weight': value.weight,
'port': value.port,
'target': target,
}
}
def _record_is_proxied(self, record):
return (
not self.cdn and
record._octodns.get('cloudflare', {}).get('proxied', False)
)
def _gen_data(self, record):
name = record.fqdn[:-1]
_type = record._type
ttl = max(self.MIN_TTL, record.ttl)
# Cloudflare supports ALIAS semantics with a root CNAME
if _type == 'ALIAS':
_type = 'CNAME'
contents_for = getattr(self, '_contents_for_{}'.format(_type))
for content in contents_for(record):
content.update({
'name': name,
'type': _type,
'ttl': ttl,
})
if _type in _PROXIABLE_RECORD_TYPES:
content.update({
'proxied': self._record_is_proxied(record)
})
yield content
def _gen_key(self, data):
# Note that most CF record data has a `content` field the value of
# which is a unique/hashable string for the record's. It includes all
# the "value" bits, but not the secondary stuff like TTL's. E.g. for
# an A it'll include the value, for a CAA it'll include the flags, tag,
# and value, ... We'll take advantage of this to try and match up old &
# new records cleanly. In general when there are multiple records for a
# name & type each will have a distinct/consistent `content` that can
# serve as a unique identifier.
# BUT... there are exceptions. MX, CAA, LOC and SRV don't have a simple
# content as things are currently implemented so we need to handle
# those explicitly and create unique/hashable strings for them.
_type = data['type']
if _type == 'MX':
return '{priority} {content}'.format(**data)
elif _type == 'CAA':
data = data['data']
return '{flags} {tag} {value}'.format(**data)
elif _type == 'SRV':
data = data['data']
return '{port} {priority} {target} {weight}'.format(**data)
elif _type == 'LOC':
data = data['data']
loc = (
'{lat_degrees}',
'{lat_minutes}',
'{lat_seconds}',
'{lat_direction}',
'{long_degrees}',
'{long_minutes}',
'{long_seconds}',
'{long_direction}',
'{altitude}',
'{size}',
'{precision_horz}',
'{precision_vert}')
return ' '.join(loc).format(**data)
return data['content']
def _apply_Create(self, change):
new = change.new
zone_id = self.zones[new.zone.name]
path = '/zones/{}/dns_records'.format(zone_id)
for content in self._gen_data(new):
self._try_request('POST', path, data=content)
def _apply_Update(self, change):
zone = change.new.zone
zone_id = self.zones[zone.name]
hostname = zone.hostname_from_fqdn(change.new.fqdn[:-1])
_type = change.new._type
existing = {}
# Find all of the existing CF records for this name & type
for record in self.zone_records(zone):
name = zone.hostname_from_fqdn(record['name'])
# Use the _record_for so that we include all of standard
# conversion logic
r = self._record_for(zone, name, record['type'], [record], True)
if hostname == r.name and _type == r._type:
# Round trip the single value through a record to contents flow
# to get a consistent _gen_data result that matches what
# went in to new_contents
data = next(self._gen_data(r))
# Record the record_id and data for this existing record
key = self._gen_key(data)
existing[key] = {
'record_id': record['id'],
'data': data,
}
# Build up a list of new CF records for this Update
new = {
self._gen_key(d): d for d in self._gen_data(change.new)
}
# OK we now have a picture of the old & new CF records, our next step
# is to figure out which records need to be deleted
deletes = {}
for key, info in existing.items():
if key not in new:
deletes[key] = info
# Now we need to figure out which records will need to be created
creates = {}
# And which will be updated
updates = {}
for key, data in new.items():
if key in existing:
# To update we need to combine the new data and existing's
# record_id. old_data is just for debugging/logging purposes
old_info = existing[key]
updates[key] = {
'record_id': old_info['record_id'],
'data': data,
'old_data': old_info['data'],
}
else:
creates[key] = data
# To do this as safely as possible we'll add new things first, update
# existing things, and then remove old things. This should (try) and
# ensure that we have as many value CF records in their system as
# possible at any given time. Ideally we'd have a "batch" API that
# would allow create, delete, and upsert style stuff so operations
# could be done atomically, but that's not available so we made the
# best of it...
# However, there are record types like CNAME that can only have a
# single value. B/c of that our create and then delete approach isn't
# actually viable. To address this we'll convert as many creates &
# deletes as we can to updates. This will have a minor upside of
# resulting in fewer ops and in the case of things like CNAME where
# there's a single create and delete result in a single update instead.
create_keys = sorted(creates.keys())
delete_keys = sorted(deletes.keys())
for i in range(0, min(len(create_keys), len(delete_keys))):
create_key = create_keys[i]
create_data = creates.pop(create_key)
delete_info = deletes.pop(delete_keys[i])
updates[create_key] = {
'record_id': delete_info['record_id'],
'data': create_data,
'old_data': delete_info['data'],
}
# The sorts ensure a consistent order of operations, they're not
# otherwise required, just makes things deterministic
# Creates
path = '/zones/{}/dns_records'.format(zone_id)
for _, data in sorted(creates.items()):
self.log.debug('_apply_Update: creating %s', data)
self._try_request('POST', path, data=data)
# Updates
for _, info in sorted(updates.items()):
record_id = info['record_id']
data = info['data']
old_data = info['old_data']
path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
self.log.debug('_apply_Update: updating %s, %s -> %s',
record_id, data, old_data)
self._try_request('PUT', path, data=data)
# Deletes
for _, info in sorted(deletes.items()):
record_id = info['record_id']
old_data = info['data']
path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
self.log.debug('_apply_Update: removing %s, %s', record_id,
old_data)
self._try_request('DELETE', path)
def _apply_Delete(self, change):
existing = change.existing
existing_name = existing.fqdn[:-1]
# Make sure to map ALIAS to CNAME when looking for the target to delete
existing_type = 'CNAME' if existing._type == 'ALIAS' \
else existing._type
for record in self.zone_records(existing.zone):
if existing_name == record['name'] and \
existing_type == record['type']:
path = '/zones/{}/dns_records/{}'.format(record['zone_id'],
record['id'])
self._try_request('DELETE', path)
def _apply(self, plan):
desired = plan.desired
changes = plan.changes
self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
len(changes))
name = desired.name
if name not in self.zones:
self.log.debug('_apply: no matching zone, creating')
data = {
'name': name[:-1],
'jump_start': False,
}
resp = self._try_request('POST', '/zones', data=data)
zone_id = resp['result']['id']
self.zones[name] = zone_id
self._zone_records[name] = {}
# Force the operation order to be Delete() -> Create() -> Update()
# This will help avoid problems in updating a CNAME record into an
# A record and vice-versa
changes.sort(key=self._change_keyer)
for change in changes:
class_name = change.__class__.__name__
getattr(self, '_apply_{}'.format(class_name))(change)
# clear the cache
self._zone_records.pop(name, None)
def _extra_changes(self, existing, desired, changes):
extra_changes = []
existing_records = {r: r for r in existing.records}
changed_records = {c.record for c in changes}
for desired_record in desired.records:
existing_record = existing_records.get(desired_record, None)
if not existing_record: # Will be created
continue
elif desired_record in changed_records: # Already being updated
continue
if (self._record_is_proxied(existing_record) !=
self._record_is_proxied(desired_record)):
extra_changes.append(Update(existing_record, desired_record))
return extra_changes