From 49c732ba95108a583e8f978492da3e486518a7ad Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 12 Sep 2023 10:12:50 -0700 Subject: [PATCH] use PowerDNS for examples so we have full/real behavior --- .gitignore | 1 + examples/README.md | 18 ++ examples/basic/README.md | 49 ++-- examples/basic/config/octodns.yaml | 30 +- examples/basic/requirements.txt | 1 + examples/docker-compose.yml | 29 ++ examples/env.sh | 1 + examples/migrating-to-octodns/README.md | 258 +++++++++--------- .../migrating-to-octodns/config/octodns.yaml | 36 +-- .../populate/my-dumpable.com.yaml | 41 +++ .../populate/octodns.yaml | 26 ++ .../populate/unused-dumpable.com.yaml | 5 + .../migrating-to-octodns/requirements.txt | 1 + .../target/my-domain.com.yaml | 64 ----- .../target/unused-domain.io.yaml | 2 - 15 files changed, 316 insertions(+), 246 deletions(-) create mode 100644 examples/docker-compose.yml create mode 100644 examples/env.sh create mode 100644 examples/migrating-to-octodns/populate/my-dumpable.com.yaml create mode 100644 examples/migrating-to-octodns/populate/octodns.yaml create mode 100644 examples/migrating-to-octodns/populate/unused-dumpable.com.yaml delete mode 100644 examples/migrating-to-octodns/target/my-domain.com.yaml delete mode 100644 examples/migrating-to-octodns/target/unused-domain.io.yaml diff --git a/.gitignore b/.gitignore index 5192821..8bf8edf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ coverage.xml dist/ env/ +examples/migrating-to-octodns/config/ htmlcov/ nosetests.xml octodns.egg-info/ diff --git a/examples/README.md b/examples/README.md index a500f44..7c0f6ce 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,3 +3,21 @@ Examples: * Getting started with a [basic octoDNS configuration](basic/) - new to octoDNS this is the place to start. It'll walk you through the main pieces of DNS IaC with octoDNS including the process of planning and applying changes. + +### Running PowerDNS + +If you'd like to play around with running the examples in this directory +interactively you'll need a target for pushing data to. +[octodns-powerdns](https://github.com/octodns/octodns-powerdns) is the best +stand-alone option for this and the examples directory makes extensive use of +it. There is a [docker-compose.yml](docker-compose.yml) file that should get a +fully functional copy of PowerDNS backed my MySQL with the API enabled along +with other relivant functionality. For any of the examples that refer to the +local PowerDNS instance the following instructions below should get it up and +running. + +1. If you haven't already [install docker compose](https://docs.docker.com/compose/install/) +1. If you don't already have a copy of octoDNS checked out run `git clone https://github.com/octodns/octodns.git` +1. In a seperate terminal window or tab +1. cd into the examples directory `cd octodns/examples` +1. Run docker-compose up `docker-compose up`, this will start up MySQL and PowerDNS running them in the foreground with their logs printing to the terminal diff --git a/examples/basic/README.md b/examples/basic/README.md index 81405b3..131ccee 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -3,7 +3,8 @@ This is the starting point octoDNS config, it's pretty similar to what you might see for managing a set of personal domains or a small business. -Most of the actual documentation for this example is in comments in the YAML configuration files. +Most of the actual documentation for this example is found in the comments in +the YAML configuration files. * [config/octodns.yaml](config/octodns.yaml) * [config/my-domain.com.yaml](config/my-domain.com.yaml) @@ -21,10 +22,14 @@ to get a copy of the example files. $ git clone https://github.com/octodns/octodns.git $ cd octodns/examples/basic/ $ python3 -mvenv env +$ source ../env.sh $ source env/bin/activate (env) $ pip install -r requirements.txt ``` +Finally check out [Running PowerDNS](../README.md#running-powerdns) to get a local +instance of PowerDNS up and going before continuing. + ## Running octoDNS the first time Once you have your configuration files and octoDNS installed you're ready to @@ -54,26 +59,26 @@ run the sync command to get it to plan an initial set of changes. 2023-08-23T15:09:51 [4577488384] INFO Manager sync: targets=['yaml'] 2023-08-23T15:09:51 [4577488384] INFO YamlProvider[config] populate: found 9 records, exists=False 2023-08-23T15:09:51 [4577488384] INFO SpfSource[no-mail] populate: found 0 records, exists=False -2023-08-23T15:09:51 [4577488384] INFO YamlProvider[yaml] plan: desired=my-domain.com. -2023-08-23T15:09:51 [4577488384] WARNING YamlProvider[yaml] root NS record supported, but no record is configured for my-domain.com. -2023-08-23T15:09:51 [4577488384] INFO YamlProvider[yaml] plan: Creates=9, Updates=0, Deletes=0, Existing Records=0 +2023-08-23T15:09:51 [4577488384] INFO PowerDnsProvider[powerdns] plan: desired=my-domain.com. +2023-08-23T15:09:51 [4577488384] WARNING PowerDnsProvider[powerdns] root NS record supported, but no record is configured for my-domain.com. +2023-08-23T15:09:51 [4577488384] INFO PowerDnsProvider[powerdns] plan: Creates=9, Updates=0, Deletes=0, Existing Records=0 2023-08-23T15:09:51 [4577488384] INFO YamlProvider[config] populate: found 0 records, exists=False 2023-08-23T15:09:51 [4577488384] INFO SpfSource[no-mail] populate: found 1 records, exists=False -2023-08-23T15:09:51 [4577488384] INFO YamlProvider[yaml] plan: desired=unused-domain.io. -2023-08-23T15:09:51 [4577488384] WARNING YamlProvider[yaml] root NS record supported, but no record is configured for unused-domain.io. -2023-08-23T15:09:51 [4577488384] INFO YamlProvider[yaml] plan: Creates=1, Updates=0, Deletes=0, Existing Records=0 +2023-08-23T15:09:51 [4577488384] INFO PowerDnsProvider[powerdns] plan: desired=unused-domain.io. +2023-08-23T15:09:51 [4577488384] WARNING PowerDnsProvider[powerdns] root NS record supported, but no record is configured for unused-domain.io. +2023-08-23T15:09:51 [4577488384] INFO PowerDnsProvider[powerdns] plan: Creates=1, Updates=0, Deletes=0, Existing Records=0 2023-08-23T15:09:51 [4577488384] INFO Plan ******************************************************************************** * unused-domain.io. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create () * Summary: Creates=1, Updates=0, Deletes=0, Existing Records=0 ******************************************************************************** * my-domain.com. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create (config) * Create (config) @@ -102,7 +107,7 @@ support managing the root NS records, some just hard code their own name servers. ```console -2023-08-23T15:09:51 [4577488384] WARNING YamlProvider[yaml] root NS record supported, but no record is configured for my-domain.com. +2023-08-23T15:09:51 [4577488384] WARNING PowerDnsProvider[powerdns] root NS record supported, but no record is configured for my-domain.com. ``` ### The plan output @@ -111,14 +116,14 @@ servers. ******************************************************************************** * unused-domain.io. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create () * Summary: Creates=1, Updates=0, Deletes=0, Existing Records=0 ******************************************************************************** * my-domain.com. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create (config) * Create (config) @@ -141,14 +146,14 @@ servers. ******************************************************************************** * unused-domain.io. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create () * Summary: Creates=1, Updates=0, Deletes=0, Existing Records=0 ******************************************************************************** * my-domain.com. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Create (config) * Create (config) @@ -162,13 +167,13 @@ servers. * Summary: Creates=9, Updates=0, Deletes=0, Existing Records=0 ******************************************************************************** -2023-08-23T15:17:00 [4671815168] INFO YamlProvider[yaml] apply: making 1 changes to unused-domain.io. -2023-08-23T15:17:00 [4671815168] INFO YamlProvider[yaml] apply: making 9 changes to my-domain.com. +2023-08-23T15:17:00 [4671815168] INFO PowerDnsProvider[powerdns] apply: making 1 changes to unused-domain.io. +2023-08-23T15:17:00 [4671815168] INFO PowerDnsProvider[powerdns] apply: making 9 changes to my-domain.com. 2023-08-23T15:17:00 [4671815168] INFO Manager sync: 10 total changes ``` ```console -(env) coho:basic ross$ octodns-sync --config-file=config/octodns.yaml +(env) $ octodns-sync --config-file=config/octodns.yaml ******************************************************************************** No changes were planned ******************************************************************************** @@ -189,12 +194,12 @@ Just like during the first run section above we'll first run octoDNS to see what We'd skim the log lines again looking for unexpected WARNINGS and then take a look at the changes. ```console -(env) coho:basic ross$ octodns-sync --config-file=config/octodns.yaml +(env) $ octodns-sync --config-file=config/octodns.yaml ... ******************************************************************************** * my-domain.com. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Update * -> @@ -217,12 +222,12 @@ Here we only see the expected changes so we're good to move forward with applying them. ```console -(env) coho:basic ross$ octodns-sync --config-file=config/octodns.yaml --doit +(env) $ octodns-sync --config-file=config/octodns.yaml --doit ... ******************************************************************************** * my-domain.com. ******************************************************************************** -* yaml (YamlProvider) +* powerdns (PowerDnsProvider) * Create Zone * Update * -> @@ -237,7 +242,7 @@ applying them. If we want we can run another plan to make sure there are no further pending changes. ```console -(env) coho:basic ross$ octodns-sync --config-file=config/octodns.yaml +(env) $ octodns-sync --config-file=config/octodns.yaml ******************************************************************************** No changes were planned ******************************************************************************** diff --git a/examples/basic/config/octodns.yaml b/examples/basic/config/octodns.yaml index bf70655..02d1148 100644 --- a/examples/basic/config/octodns.yaml +++ b/examples/basic/config/octodns.yaml @@ -18,22 +18,16 @@ providers: no-mail: class: octodns_spf.SpfSource - # This is just a placeholder for the purposes of this example, in your - # configuration you would replace it with configuration for the DNS - # provider(s) you wish to use, e.g. Route53, NS1, Azure DNS, ... See - # https://github.com/octodns/octodns#providers for a list of supported - # providers and have a look in each provider's repo for details on installing - # and configuring the provider. Convention is to name the provider with the - # unique portion of the provider class/company name in all lowercase, here - # YamlProvider -> yaml. Route53 would be route53, NS1 -> ns1, etc. - yaml: - class: octodns.provider.yaml.YamlProvider - directory: ./target - # This is a rarely used option that tells the YamlProvider that it should - # load existing records from disk rather than assume nothing exists - # created every a plan is made. - # TODO: - #populate_existing: true + # PowerDNS serves as a good option for a locally runnable standalone provider + # to use in our examples. See the top-level example README for information on + # how to run it and the documentation for octodns-powerdns for documentation + # about the details of the config below. + powerdns: + class: octodns_powerdns.PowerDnsProvider + host: 127.0.0.1 + port: 8081 + api_key: env/POWERDNS_API_KEY + timeout: 10 # If you wish to push your record data to multiple providers you'd define the # other here and include them in the targets list down below. See the @@ -60,5 +54,5 @@ zones: - no-mail # This is the place(s) we push the record data to when applying changes targets: - # Here we're pushing things to our example provider. - - yaml + # Here we're pushing things to our local PowerDNS instance + - powerdns diff --git a/examples/basic/requirements.txt b/examples/basic/requirements.txt index b30fe47..00239f7 100644 --- a/examples/basic/requirements.txt +++ b/examples/basic/requirements.txt @@ -1,2 +1,3 @@ octodns>=1.0.1 +octodns_powerdns>=0.0.5 octodns_spf>=0.0.2 diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..599c751 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3.9" # optional since v1.27.0 + +volumes: + powerdns-db-data: + name: powerdns-db-data + +services: + db: + image: mariadb:10.1 + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: l3tmein + volumes: + - powerdns-db-data:/var/lib/mysql + powerdns: + image: psitrax/powerdns + ports: + - "53:53/tcp" + - "53:53/udp" + - "8081:8081" + environment: + MYSQL_HOST: db + MYSQL_PORT: 3306 + MYSQL_USER: root + MYSQL_PASS: l3tmein + depends_on: + - db + command: --api=yes --api-key=its@secret --loglevel=99 --webserver=yes --webserver-address=0.0.0.0 --webserver-allow-from=0.0.0.0/0 --webserver-password=its@secret --webserver-port=8081 --enable-lua-records=shared --edns-subnet-processing=yes diff --git a/examples/env.sh b/examples/env.sh new file mode 100644 index 0000000..7501e7c --- /dev/null +++ b/examples/env.sh @@ -0,0 +1 @@ +export POWERDNS_API_KEY="its@secret" diff --git a/examples/migrating-to-octodns/README.md b/examples/migrating-to-octodns/README.md index d058853..cd3d04d 100644 --- a/examples/migrating-to-octodns/README.md +++ b/examples/migrating-to-octodns/README.md @@ -3,7 +3,8 @@ Importing an existing DNS setup into octoDNS management is a very straightforward process and can generally be completed in minutes. -Some relevant documentation for this example is in comments in the YAML configuration files. +Some relevant documentation for this example is in comments in the YAML +configuration files. * [config/octodns.yaml](config/octodns.yaml) * [config/my-domain.com.yaml](config/my-domain.com.yaml) @@ -22,6 +23,7 @@ to get a copy of the example files. $ git clone https://github.com/octodns/octodns.git $ cd octodns/examples/basic/ $ python3 -mvenv env +$ source ../env.sh $ source env/bin/activate (env) $ pip install -r requirements.txt ``` @@ -44,6 +46,22 @@ $ git commit -m "Importting existing records into octoDNS" $ git push -u origin main ``` +## Populating some data that we can later dump + +We need zone & record data in our provider in order to have something to dump +out into a YAML config, thus our first step is to populate PowerDNS. This is +just to have something to work with the the actual process that begins in the +next step. It's not something you'd normally do when migrating to octoDNS. + +Step 0 is to get a local PowerDNS instance running. +Check out out [Running PowerDNS](../README.md#running-powerdns) for info on +starting that up. Once you've done that run the following. You can ignore the +output and move on to the next step. + +```console +(env) $ octodns-sync --config-file=populate/octodns.yaml --doit +``` + ## Running octodns-dump Once you have your configuration files and octoDNS installed you're ready to @@ -52,52 +70,61 @@ dump your zone configs. Here we've assumed that the provider being used supports [dynamic-zone-config](../dynamic-zone-config) for details on how to explicitly list your zones in the config file. -The quoted * below is what tells octodns-dump to dynamically source the list of -zones and dump config files for all of them. If you would only like to dump a -specific zone you can replace it with the zone name, e.g. `my-domain.com.`, -making sure to include the trailing `.` +We first tell octodns-dump where to find our config file. We then tell it that +we want to use the output provider defined in that file named `config` rather +than having a default one auto-created. This would allow us to customize the +details of the provider, e.g. things like the `default_ttl`. +We'll then tell dump where we would like it to write our zone files, in this +case we want it to put them into `config` provider's directory `./config`. + +The quoted * tells octodns-dump to dynamically source the list of zones and +dump config files for all of them. If you would only like to dump a specific +zone you can replace it with the zone name, e.g. `my-domain.com.`, making sure +to include the trailing `.` + +The final argument passed to `octodns-dump` is the source of data you'd like to +dump. Here that's our existing provider `powerdns`. + +Note that if you're previously run other examples you may see entries in the +log output for those zones in addition to the ones we populated for this +exercise. They shouldn't cause any problems and can be ignored. ```console -(env) coho:migrating-to-octodns ross$ octodns-dump --config-file=config/octodns.yaml --output-dir config/ '*' some-provider -2023-08-25T10:36:00 [4413005312] INFO Manager __init__: config_file=config/octodns.yaml, (octoDNS 1.0.0+6e7f036b) -2023-08-25T10:36:00 [4413005312] INFO Manager _config_executor: max_workers=1 -2023-08-25T10:36:00 [4413005312] INFO Manager _config_include_meta: include_meta=False -2023-08-25T10:36:00 [4413005312] INFO Manager _config_auto_arpa: auto_arpa=False -2023-08-25T10:36:00 [4413005312] INFO Manager __init__: global_processors=[] -2023-08-25T10:36:00 [4413005312] INFO Manager __init__: provider=config (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T10:36:00 [4413005312] INFO Manager __init__: provider=some-provider (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T10:36:00 [4413005312] INFO Manager dump: zone=*, output_dir=config/, output_provider=None, lenient=False, split=False, sources=['some-provider'] -2023-08-25T10:36:00 [4413005312] INFO Manager dump: using custom YamlProvider -2023-08-25T10:36:00 [4413005312] INFO Manager sync: dynamic zone=*, sources=[YamlProvider] -2023-08-25T10:36:00 [4413005312] INFO Manager sync: adding dynamic zone=my-domain.com. -2023-08-25T10:36:00 [4413005312] INFO Manager sync: adding dynamic zone=unused-domain.io. +(env) $ octodns-dump --config-file=config/octodns.yaml --output-provider config --output-dir ./config '*' powerdns +2023-09-12T09:39:59 [4580544000] INFO Manager __init__: config_file=config/octodns.yaml, (octoDNS 1.0.0+bca53089) +2023-09-12T09:39:59 [4580544000] INFO Manager _config_executor: max_workers=1 +2023-09-12T09:39:59 [4580544000] INFO Manager _config_include_meta: include_meta=False +2023-09-12T09:39:59 [4580544000] INFO Manager _config_auto_arpa: auto_arpa=False +2023-09-12T09:39:59 [4580544000] INFO Manager __init__: global_processors=[] +2023-09-12T09:39:59 [4580544000] INFO Manager __init__: provider=config (octodns.provider.yaml 1.0.0+bca53089) +2023-09-12T09:40:00 [4580544000] INFO Manager __init__: provider=powerdns (octodns_powerdns 0.0.4+3189e9c2) +2023-09-12T09:40:00 [4580544000] INFO Manager dump: zone=*, output_dir=./config, output_provider=config, lenient=False, split=False, sources=['powerdns'] +2023-09-12T09:40:00 [4580544000] INFO Manager dump: using specified output_provider=config +2023-09-12T09:40:00 [4580544000] INFO Manager sync: dynamic zone=*, sources=[PowerDnsProvider] +2023-09-12T09:40:00 [4580544000] INFO Manager sync: adding dynamic zone=my-dumpable.com. +2023-09-12T09:40:00 [4580544000] INFO Manager sync: adding dynamic zone=unused-domain.io. Traceback (most recent call last): File "/Users/ross/octodns/octodns/examples/migrating-to-octodns/env/bin/octodns-dump", line 8, in sys.exit(main()) ^^^^^^ - File "/Users/ross/octodns/octodns/octodns/cmds/dump.py", line 51, in main + File "/Users/ross/octodns/octodns/examples/migrating-to-octodns/env/lib/python3.11/site-packages/octodns/cmds/dump.py", line 51, in main manager.dump( - File "/Users/ross/octodns/octodns/octodns/manager.py", line 868, in dump + File "/Users/ross/octodns/octodns/examples/migrating-to-octodns/env/lib/python3.11/site-packages/octodns/manager.py", line 868, in dump source.populate(zone, lenient=lenient) - File "/Users/ross/octodns/octodns/octodns/provider/yaml.py", line 229, in populate - self._populate_from_file(filename, zone, lenient) - File "/Users/ross/octodns/octodns/octodns/provider/yaml.py", line 175, in _populate_from_file + File "/Users/ross/octodns/octodns/examples/migrating-to-octodns/env/lib/python3.11/site-packages/octodns_powerdns/__init__.py", line 433, in populate record = Record.new( ^^^^^^^^^^^ - File "/Users/ross/octodns/octodns/octodns/record/base.py", line 76, in new + File "/Users/ross/octodns/octodns/examples/migrating-to-octodns/env/lib/python3.11/site-packages/octodns/record/base.py", line 76, in new raise ValidationError(fqdn, reasons, context) -octodns.record.exception.ValidationError: Invalid record "my-domain.com.", ./target/my-domain.com.yaml, line 17, column 5 - - NS value "ns1.some-provider.com" missing trailing . - - NS value "ns2.some-provider.com" missing trailing . - - NS value "ns3.some-provider.com" missing trailing . - - NS value "ns4.some-provider.com" missing trailing . +octodns.record.exception.ValidationError: Invalid record "sshfp.my-dumpable.com." + - unrecognized algorithm "42" + - unrecognized algorithm "43" ``` We've intentionally configured data in the provider without following (octoDNS's) best practices as it's common to run across these sorts of things -when migrating. In this case the NS values are missing their trailing `.`, -`ns1.some-provider.com` should be `ns1.some-provider.com.`. +when migrating. In this case there is a SSHFP record with invalid `algorithm`s. Regardless of whether or not you hit errors it is important to carefully look over the output of octodns-dump to ensure that it was able to make sense of and @@ -112,76 +139,86 @@ this situation see [Dynamic Records](/docs/dynamic_records.md). ## Running octodns-dump again, now with --lenient -Back to our example, the missing `.`s aren't a huge deal and octoDNS can be -asked to ignore them with the `--lenient` argument. +Back to our example, the invalid `algorithm`s aren't a huge deal and octoDNS +can be asked to ignore them with the `--lenient` argument. ```console -(env) coho:migrating-to-octodns ross$ octodns-dump --config-file=config/octodns.yaml --output-dir config/ '*' some-provider --lenient -2023-08-25T10:43:26 [4528057856] INFO Manager __init__: config_file=config/octodns.yaml, (octoDNS 1.0.0+6e7f036b) -2023-08-25T10:43:26 [4528057856] INFO Manager _config_executor: max_workers=1 -2023-08-25T10:43:26 [4528057856] INFO Manager _config_include_meta: include_meta=False -2023-08-25T10:43:26 [4528057856] INFO Manager _config_auto_arpa: auto_arpa=False -2023-08-25T10:43:26 [4528057856] INFO Manager __init__: global_processors=[] -2023-08-25T10:43:26 [4528057856] INFO Manager __init__: provider=config (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T10:43:26 [4528057856] INFO Manager __init__: provider=some-provider (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T10:43:26 [4528057856] INFO Manager dump: zone=*, output_dir=config/, output_provider=None, lenient=True, split=False, sources=['some-provider'] -2023-08-25T10:43:26 [4528057856] INFO Manager dump: using custom YamlProvider -2023-08-25T10:43:26 [4528057856] INFO Manager sync: dynamic zone=*, sources=[YamlProvider] -2023-08-25T10:43:26 [4528057856] INFO Manager sync: adding dynamic zone=my-domain.com. -2023-08-25T10:43:26 [4528057856] INFO Manager sync: adding dynamic zone=unused-domain.io. -2023-08-25T10:43:26 [4528057856] WARNING Record Invalid record "my-domain.com.", ./target/my-domain.com.yaml, line 17, column 5 - - NS value "ns1.some-provider.com" missing trailing . - - NS value "ns2.some-provider.com" missing trailing . - - NS value "ns3.some-provider.com" missing trailing . - - NS value "ns4.some-provider.com" missing trailing . -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[some-provider] populate: found 10 records, exists=False -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] plan: desired=my-domain.com. -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] plan: Creates=10, Updates=0, Deletes=0, Existing Records=0 -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] apply: making 10 changes to my-domain.com. -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[some-provider] populate: found 0 records, exists=False -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] plan: desired=unused-domain.io. -2023-08-25T10:43:26 [4528057856] WARNING YamlProvider[dump] root NS record supported, but no record is configured for unused-domain.io. -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] plan: No changes -2023-08-25T10:43:26 [4528057856] INFO YamlProvider[dump] apply: making 0 changes to unused-domain.io. +(env) $ octodns-dump --config-file=config/octodns.yaml --output-provider config --output-dir ./config '*' powerdns --lenient +2023-09-12T09:47:48 [4566433280] INFO Manager __init__: config_file=config/octodns.yaml, (octoDNS 1.0.0+bca53089) +2023-09-12T09:47:48 [4566433280] INFO Manager _config_executor: max_workers=1 +2023-09-12T09:47:48 [4566433280] INFO Manager _config_include_meta: include_meta=False +2023-09-12T09:47:48 [4566433280] INFO Manager _config_auto_arpa: auto_arpa=False +2023-09-12T09:47:48 [4566433280] INFO Manager __init__: global_processors=[] +2023-09-12T09:47:48 [4566433280] INFO Manager __init__: provider=config (octodns.provider.yaml 1.0.0+bca53089) +2023-09-12T09:47:48 [4566433280] INFO Manager __init__: provider=powerdns (octodns_powerdns 0.0.4+3189e9c2) +2023-09-12T09:47:48 [4566433280] INFO Manager dump: zone=*, output_dir=./config, output_provider=config, lenient=True, split=False, sources=['powerdns'] +2023-09-12T09:47:48 [4566433280] INFO Manager dump: using specified output_provider=config +2023-09-12T09:47:48 [4566433280] INFO Manager sync: dynamic zone=*, sources=[PowerDnsProvider] +2023-09-12T09:47:48 [4566433280] INFO Manager sync: adding dynamic zone=my-dumpable.com. +2023-09-12T09:47:48 [4566433280] INFO Manager sync: adding dynamic zone=unused-dumpable.com. +2023-09-12T09:47:48 [4566433280] WARNING Record Invalid record "sshfp.my-dumpable.com." + - unrecognized algorithm "42" + - unrecognized algorithm "43" +2023-09-12T09:47:48 [4566433280] INFO PowerDnsProvider[powerdns] populate: found 8 records, exists=True +2023-09-12T09:47:48 [4566433280] INFO YamlProvider[config] plan: desired=my-dumpable.com. +2023-09-12T09:47:48 [4566433280] INFO YamlProvider[config] plan: Creates=8, Updates=0, Deletes=0, Existing Records=0 +2023-09-12T09:47:48 [4566433280] INFO YamlProvider[config] apply: making 8 changes to my-dumpable.com. +2023-09-12T09:47:48 [4566433280] INFO PowerDnsProvider[powerdns] populate: found 1 records, exists=True +2023-09-12T09:47:49 [4566433280] INFO YamlProvider[config] plan: desired=unused-dumpable.com. +2023-09-12T09:47:49 [4566433280] WARNING YamlProvider[config] root NS record supported, but no record is configured for unused-dumpable.com. +2023-09-12T09:47:49 [4566433280] INFO YamlProvider[config] plan: Creates=1, Updates=0, Deletes=0, Existing Records=0 +2023-09-12T09:47:49 [4566433280] INFO YamlProvider[config] apply: making 1 changes to unused-dumpable.com. ``` +With `--lenient` we now see a warning about the validation problems, but +octoDNS does its best to continue anyway. + ## Examining the results We can now take a look in the config/ directory to see the created zone files along side our main config. ```console -(env) coho:migrating-to-octodns ross$ ls -1 config/ -my-domain.com.yaml +(env) ls -1 config/ +my-dumpable.com.yaml octodns.yaml -unused-domain.io.yaml +unused-dumpable.com.yaml ``` -If you open up the my-domain.com.yaml file you'll note that the root NS record values are missing -their trailing `.`s. octoDNS has done its best to dump things as they currently -are in the provider. That means that if you tried to generate a plan with -`octodns-sync` now it would fail b/c the configuration doesn't follow best -practices. +Again note if you're run through other examples you may see more files for +their zones in this directory. They can be ignored for the purpose of the +example, but they will be imported and managed nonetheless. + +If you open up the my-dumpable.com.yaml file you'll note that the SSHFP +record's values have 42 and 43 for their algorithm field. + +With those values if you tried to generate a plan with `octodns-sync` now it +would fail with validation errors. When migrating it's recommended to make as few changes to your config as possible initially, that is you want to bring the setup as-is under octoDNS -management and avoid churn as much as possible. This is the safest approach, -change as little as possible and incrementally work towards having a fully -managed and compliant/best practice config. +management. This is the safest approach, change as little as possible and +incrementally work towards having a fully managed and compliant/best practice +config. -So for now we'll enable `lenient` on the root NS record. For more details on -see the [lenience](../lenience/) example. +So for now we'll enable `lenient` on the SSHFP record by editing +my-dumpable.com.yaml adding `octodns.lenient = true` as shown below. For more +details on see [lenience](/docs/records.md#lenience). ```yaml ... - - octodns: - lenient: true - type: NS - values: - - ns1.some-provider.com - - ns2.some-provider.com - - ns3.some-provider.com - - ns4.some-provider.com +sshpf: + # TODO: look into bringing this record into compliance + octodns: + lenient: true + type: SSHFP + values: + - algorithm: 42 + fingerprint: abcdef1234567890 + fingerprint_type: 1 + - algorithm: 43 + fingerprint: abcdef1234567890 + fingerprint_type: 1 ... ``` @@ -192,53 +229,30 @@ we've looked closely at the log output zone configuration files we have a pretty good idea the answer is no. ```console -(env) coho:migrating-to-octodns ross$ octodns-sync --config-file=config/octodns.yaml -2023-08-25T11:07:44 [4644843008] INFO Manager __init__: config_file=config/octodns.yaml, (octoDNS 1.0.0+6e7f036b) -2023-08-25T11:07:44 [4644843008] INFO Manager _config_executor: max_workers=1 -2023-08-25T11:07:44 [4644843008] INFO Manager _config_include_meta: include_meta=False -2023-08-25T11:07:44 [4644843008] INFO Manager _config_auto_arpa: auto_arpa=False -2023-08-25T11:07:44 [4644843008] INFO Manager __init__: global_processors=[] -2023-08-25T11:07:44 [4644843008] INFO Manager __init__: provider=config (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T11:07:44 [4644843008] INFO Manager __init__: provider=some-provider (octodns.provider.yaml 1.0.0+6e7f036b) -2023-08-25T11:07:44 [4644843008] INFO Manager sync: eligible_zones=[], eligible_targets=[], dry_run=True, force=False, plan_output_fh= -2023-08-25T11:07:44 [4644843008] INFO Manager sync: sources=['config'] -2023-08-25T11:07:44 [4644843008] INFO Manager sync: dynamic zone=*, sources=[YamlProvider] -2023-08-25T11:07:44 [4644843008] INFO Manager sync: adding dynamic zone=my-domain.com. -2023-08-25T11:07:44 [4644843008] INFO Manager sync: adding dynamic zone=unused-domain.io. -2023-08-25T11:07:44 [4644843008] INFO Manager sync: zone=my-domain.com. -2023-08-25T11:07:44 [4644843008] INFO Manager sync: sources=['config'] -2023-08-25T11:07:44 [4644843008] INFO Manager sync: targets=['some-provider'] -2023-08-25T11:07:44 [4644843008] INFO Manager sync: zone=unused-domain.io. -2023-08-25T11:07:44 [4644843008] INFO Manager sync: sources=['config'] -2023-08-25T11:07:44 [4644843008] INFO Manager sync: targets=['some-provider'] -2023-08-25T11:07:44 [4644843008] WARNING Record Invalid record "my-domain.com.", ./config/my-domain.com.yaml, line 9, column 5 - - NS value "ns1.some-provider.com" missing trailing . - - NS value "ns2.some-provider.com" missing trailing . - - NS value "ns3.some-provider.com" missing trailing . - - NS value "ns4.some-provider.com" missing trailing . -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[config] populate: found 10 records, exists=False -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] plan: desired=my-domain.com. -2023-08-25T11:07:44 [4644843008] WARNING Record Invalid record "my-domain.com.", ./target/my-domain.com.yaml, line 17, column 5 - - NS value "ns1.some-provider.com" missing trailing . - - NS value "ns2.some-provider.com" missing trailing . - - NS value "ns3.some-provider.com" missing trailing . - - NS value "ns4.some-provider.com" missing trailing . -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] populate: found 10 records, exists=False -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] plan: No changes -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[config] populate: found 0 records, exists=False -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] plan: desired=unused-domain.io. -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] populate: found 0 records, exists=False -2023-08-25T11:07:44 [4644843008] WARNING YamlProvider[some-provider] root NS record supported, but no record is configured for unused-domain.io. -2023-08-25T11:07:44 [4644843008] INFO YamlProvider[some-provider] plan: No changes -2023-08-25T11:07:44 [4644843008] INFO Plan +(env) $ octodns-sync --config-file=config/octodns.yaml +... +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] plan: desired=unused-dumpable.com. +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] populate: found 1 records, exists=True +2023-09-12T10:04:27 [4608876032] WARNING PowerDnsProvider[powerdns] root NS record supported, but no record is configured for unused-dumpable.com. +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] plan: No changes +2023-09-12T10:04:27 [4608876032] INFO YamlProvider[config] populate: found 5 records, exists=False +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] plan: desired=exxampled.com. +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] populate: found 5 records, exists=True +2023-09-12T10:04:27 [4608876032] INFO PowerDnsProvider[powerdns] plan: No changes +2023-09-12T10:04:27 [4608876032] INFO Plan ******************************************************************************** No changes were planned ******************************************************************************** ``` +Most of the output above is omitted, this is the final few lines showing that +no changes need to be made to get `powerdns` to match `config`. It does have a +warning about the lack of root NS records in `unused-dumpable.com.`, but we'll +ignore that in this example. + If you had unsupported record types or advanced features in use this may not be -the case. It's completely safe to generate a plan, octoDNS won't make any -changes. +the case and you may see changes. It's completely safe to generate a plan, +octoDNS won't make any of the changes listed. If you see things in the plan output section it's time to triple check and make sure they're, hopefully minimal, modifications your OK with making. diff --git a/examples/migrating-to-octodns/config/octodns.yaml b/examples/migrating-to-octodns/config/octodns.yaml index 0841aa8..0ad49c1 100644 --- a/examples/migrating-to-octodns/config/octodns.yaml +++ b/examples/migrating-to-octodns/config/octodns.yaml @@ -1,31 +1,31 @@ --- providers: - # See the basic example for descriptions of the bulk of this config file, here - # we'll focus on things related to octodns-dump + # See the basic example for descriptions of the basics of this config file, + # here we'll focus on things related to octodns-dump. + + # This is our config provider, it sources YAML from disk, but to start we + # don't have any files in the ./config directory. We need to dump what's in + # our provider, in this case a local PowerDNS setup, into this config + # directory as a starting point. config: class: octodns.provider.yaml.YamlProvider directory: ./config - # In a real setup this would be your existing DNS provider where all of your - # zones are configured. E.g. Route53, Azure, Cloudflare, NS1, ... Here it's - # using a second YamlProvider so that the example can be self contained and - # will work for anyone. For information on how to configure your provider see - # its README.md - some-provider: - class: octodns.provider.yaml.YamlProvider - directory: ./target - # This is a rarely used option that tells the YamlProvider that it should - # load existing records from disk rather than assume nothing exists - # created every a plan is made. - # TODO: - #populate_existing: true + # This is our existing provider, the one we want to bring under octoDNS + # management. This example will walk through dumping the existing data in + # this provider into the config provider's directory above. At that point we + # should be ready to start managing things with octoDNS going forward. + powerdns: + class: octodns_powerdns.PowerDnsProvider + host: 127.0.0.1 + port: 8081 + api_key: env/POWERDNS_API_KEY + timeout: 10 zones: - # We're using dynamic zone config to dynamically build the list of configured - # zones using the sources & targets below for each. '*': sources: - config targets: - - some-provider + - powerdns diff --git a/examples/migrating-to-octodns/populate/my-dumpable.com.yaml b/examples/migrating-to-octodns/populate/my-dumpable.com.yaml new file mode 100644 index 0000000..62482be --- /dev/null +++ b/examples/migrating-to-octodns/populate/my-dumpable.com.yaml @@ -0,0 +1,41 @@ +--- +# Just some records so we'll have something to dump, see the basic example for +# documentation/descriptions of what's in here. +'': + - type: A + values: + - 203.0.113.42 + - 203.0.113.43 + - type: AAAA + values: &WEB_AAAA_VALUES + - 2001:DB8::44 + - type: NS + values: + - ns1.some-provider.com. + - ns2.some-provider.com. + - ns3.some-provider.com. + - ns4.some-provider.com. + - type: TXT + values: + - some-verification=3becb991-932f-4433-a280-9df6f39b6194 + - z-other-thing=this proves i have control over this domain +'*': + - type: A + value: 203.0.113.45 + - type: AAAA + value: 2001:DB8::46 +sshfp: + type: SSHFP + values: + # These values won't pass validation. They'll be used to walk through the + # process of safely bringing an existing config up to octoDNS's + # recommended best practices. + - algorithm: 42 + fingerprint: abcdef1234567890 + fingerprint_type: 1 + - algorithm: 43 + fingerprint: abcdef1234567890 + fingerprint_type: 1 +www: + - type: CNAME + value: my-dumpable.com. diff --git a/examples/migrating-to-octodns/populate/octodns.yaml b/examples/migrating-to-octodns/populate/octodns.yaml new file mode 100644 index 0000000..c2ce511 --- /dev/null +++ b/examples/migrating-to-octodns/populate/octodns.yaml @@ -0,0 +1,26 @@ +--- +providers: + # This is just a quick and easy way to load some data into PowerDNS that we + # can dump while working through the example. See the basic example for + # documentation of what's in here + config: + class: octodns.provider.yaml.YamlProvider + directory: ./populate + + powerdns: + class: octodns_powerdns.PowerDnsProvider + host: 127.0.0.1 + port: 8081 + api_key: env/POWERDNS_API_KEY + timeout: 10 + +zones: + + '*': + # we're disabling strict validation here so we can load up data that won't + # meet octoDNS's best practices so that we can show + lenient: true + sources: + - config + targets: + - powerdns diff --git a/examples/migrating-to-octodns/populate/unused-dumpable.com.yaml b/examples/migrating-to-octodns/populate/unused-dumpable.com.yaml new file mode 100644 index 0000000..16c9dc3 --- /dev/null +++ b/examples/migrating-to-octodns/populate/unused-dumpable.com.yaml @@ -0,0 +1,5 @@ +--- +'': + type: TXT + values: + - v=spf1 -all diff --git a/examples/migrating-to-octodns/requirements.txt b/examples/migrating-to-octodns/requirements.txt index b30fe47..00239f7 100644 --- a/examples/migrating-to-octodns/requirements.txt +++ b/examples/migrating-to-octodns/requirements.txt @@ -1,2 +1,3 @@ octodns>=1.0.1 +octodns_powerdns>=0.0.5 octodns_spf>=0.0.2 diff --git a/examples/migrating-to-octodns/target/my-domain.com.yaml b/examples/migrating-to-octodns/target/my-domain.com.yaml deleted file mode 100644 index 8d351c8..0000000 --- a/examples/migrating-to-octodns/target/my-domain.com.yaml +++ /dev/null @@ -1,64 +0,0 @@ ---- -# These are root/APEX records, things that live at the top-level of the zone. -# There are multiple records so there's a YAML list, each element in that list -# defines its own record. -'': - # We first have an A record, mapping A my-domain.com to two IPv4 addresses, we - # also use a YAML anchor to effectively store the values for later use. - - type: A - values: &WEB_A_VALUES - - 203.0.113.42 - - 203.0.113.43 - # Similar to the A above, this is a AAAA for my-domain.com with a single IPv6 - # addresse. - - type: AAAA - values: &WEB_AAAA_VALUES - - 2001:DB8::44 - - type: NS - values: - - ns1.some-provider.com - - ns2.some-provider.com - - ns3.some-provider.com - - ns4.some-provider.com - # Finally it's common to have to prove ownership of a domain with a TXT value - # in an APEX record. The following are a couple made up examples of how that - # would be done. As you'll see when you plan things The SpfSource will merge - # its SPF value in with the values defined here - - type: TXT - values: - - some-verification=3becb991-932f-4433-a280-9df6f39b6194 - - z-other-thing=this proves i have control over this domain -# This is a wildcard record, any A or AAAA query not matching a more specific -# answer defined in this file will match this name and receive the associated -# value in response. -'*': - - type: A - # Note here we have a single value rather than values and use `value`. - value: 203.0.113.45 - - type: AAAA - value: 2001:DB8::46 -# Note that the records in this zone are sorted, by default YamlProvider -# enforces alphabetical sorting of records and even the keys within record -# data. This is an organizational best practice, but can be disabled with the -# enforce_order parameter to YamlProvider -NADCBiQKBgQ._companyname: - type: TXT - value: a-different-proof-of-ownership -# We want www and the APEX to return the same values for A/AAAA queries. We -# stored the values earlier in YAML anchors so we can use them now on. -# This is a CNAME record -pointer: - type: CNAME - # CNAMEs can only have a single value so they require `value` to be used. As a - # best practice the target name must end with a `.` If you have a specific - # case where you need to omit the `.`, or other enforced best practices, see - # the lenience example for more information. - value: look.over-here.net. -# This is an example of a record with a `.` in it's name, it would result for a -# TXT query of NADCBiQKBgQ._companyname.my-domain.com. This specific example is -# another commonly required proof of overship record. -www: - - type: A - values: *WEB_A_VALUES - - type: AAAA - values: *WEB_AAAA_VALUES diff --git a/examples/migrating-to-octodns/target/unused-domain.io.yaml b/examples/migrating-to-octodns/target/unused-domain.io.yaml deleted file mode 100644 index d513fd6..0000000 --- a/examples/migrating-to-octodns/target/unused-domain.io.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# Nothing in here, just an empty domain b/c it's not actually used for anything