mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

Fix major performance issue with add_record O(N^2)

Browse Source 
Before, 1-2k record took ~10s and more than that was just painful, 5k took
forever. This records things to keep a dict of nodes with a set of records so
that we can quickly "jump" to the point we're looking for without having to
search. 10k records now takes ~5s.
This commit is contained in:
Ross McFarland
2017-07-02 18:23:45 -07:00
parent 69817ab465
commit 908698da49
8 changed files with 44 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
octodns/zone.py
View File

@@ -5,6 +5,7 @@
from __future__ import absolute_import, division, print_function, \
unicode_literals
from collections import defaultdict
from logging import getLogger
import re
@@ -39,13 +40,19 @@ class Zone(object):
# Force everyting to lowercase just to be safe
self.name = str(name).lower() if name else name
self.sub_zones = sub_zones
self.records = set()
# We're grouping by node, it allows us to efficently search for
# duplicates and detect when CNAMEs co-exist with other records
self._records = defaultdict(set)
# optional leading . to match empty hostname
# optional trailing . b/c some sources don't have it on their fqdn
self._name_re = re.compile('\.?{}?$'.format(name))
self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones)
@property
def records(self):
return set([r for _, node in self._records.items() for r in node])
def hostname_from_fqdn(self, fqdn):
return self._name_re.sub('', fqdn)
@@ -53,9 +60,6 @@ class Zone(object):
name = record.name
last = name.split('.')[-1]
if replace and record in self.records:
self.records.remove(record)
if last in self.sub_zones:
if name != last:
# it's a record for something under a sub-zone
@@ -67,19 +71,30 @@ class Zone(object):
raise SubzoneRecordException('Record {} a managed sub-zone '
'and not of type NS'
.format(record.fqdn))
# TODO: this is pretty inefficent
for existing in self.records:
if record == existing:
raise DuplicateRecordException('Duplicate record {}, type {}'
.format(record.fqdn,
record._type))
elif name == existing.name and (record._type == 'CNAME' or
existing._type == 'CNAME'):
raise InvalidNodeException('Invalid state, CNAME at {} '
'cannot coexist with other records'
.format(record.fqdn))
self.records.add(record)
if replace:
# will remove it if it exists
self._records[name].discard(record)
node = self._records[name]
if record in node:
# We already have a record at this node of this type
raise DuplicateRecordException('Duplicate record {}, type {}'
.format(record.fqdn,
record._type))
elif ((record._type == 'CNAME' and len(node) > 0) or
('CNAME' in map(lambda r: r._type, node))):
# We're adding a CNAME to existing records or adding to an existing
# CNAME
raise InvalidNodeException('Invalid state, CNAME at {} cannot '
'coexist with other records'
.format(record.fqdn))
node.add(record)
def _remove_record(self, record):
'Only for use in tests'
self._records[record.name].discard(record)
def changes(self, desired, target):
self.log.debug('changes: zone=%s, target=%s', self, target)