From 9d2ee62dd3d3c44aebcbbe2116a9de7213444eb1 Mon Sep 17 00:00:00 2001 From: Stefan Pratter Date: Fri, 9 Nov 2018 07:38:15 +0000 Subject: [PATCH 1/2] fix init.sql --- config/facsimile/tmpl/_ALL_/init.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/facsimile/tmpl/_ALL_/init.sql b/config/facsimile/tmpl/_ALL_/init.sql index d026d702..a21edb31 100644 --- a/config/facsimile/tmpl/_ALL_/init.sql +++ b/config/facsimile/tmpl/_ALL_/init.sql @@ -2,18 +2,18 @@ {% for k, each in module.iteritems() %} {% if each.db %} {% if each.db.name %} -create database if not exists {{env.rc.db.prefix}}{{each.db.name}} character set = utf8; -grant all on {{env.rc.db.prefix}}{{each.db.name}}.* to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; -grant all on {{env.rc.db.prefix}}{{each.db.name}}.* to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; +create database if not exists {{env.rc.db.default.prefix}}{{each.db.name}} character set = utf8; +grant all on {{env.rc.db.default.prefix}}{{each.db.name}}.* to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; +grant all on {{env.rc.db.default.prefix}}{{each.db.name}}.* to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; {% endif %} {% for table in each.db.selectable %} -grant select on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; -grant select on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; +grant select on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; +grant select on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; {% endfor %} {% for table in each.db.writable %} -grant all on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; -grant all on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; +grant all on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}'; +grant all on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}'; {% endfor %} {% endif %} From e3ab24d19da54d6745c4ce321b597be58c1c1c60 Mon Sep 17 00:00:00 2001 From: Stefan Pratter Date: Fri, 9 Nov 2018 09:56:16 +0000 Subject: [PATCH 2/2] how to populate your test instance with initial data --- docs/deploy.md | 49 ++++++-- .../management/commands/pdb_load_data.py | 110 ++++++++++++++++++ 2 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 peeringdb_server/management/commands/pdb_load_data.py diff --git a/docs/deploy.md b/docs/deploy.md index e09e99a3..57c738c4 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -9,16 +9,6 @@ This document uses the following variables export PDB_REPO=git@github.com:peeringdb/peeringdb.git ``` -### Install obfuscation tools (Only needed if you want to obfuscate js) - -```sh -git clone src.20c.com:20c/sys-deploy -mkdir -p ~/.local/google -wget https://dl.google.com/closure-compiler/compiler-latest.zip -unzip compiler-latest.zip -mv compiler.jar ~/.local/google -``` - ### Install facsimile ```sh @@ -108,6 +98,45 @@ mysql -u root -p < .facsimile/tmp/RELEASE/dev/peeringdb/init.sql ./manage.py runserver ``` +## Populating with data + +There are currently 2 ways to get some data into your developer instance + +1) Sync from production peeringdb - slow, but accurate data +2) Generate test data - fast, but marginally useful test data + +### 1) Sync from Production + +You can populate your data from peeringdb.com using + +```sh +./manage.py pdb_load_data +``` + +However be prepared for this to take 15-20 minutes as it will not only sync the entities, but also set up usergroups for each organization and so forth. + +This can only be used to populate initial data. Once you have started adding / updating objects and your data diverges from production data, it is no longer useful to call this command nor will it allow you to. + +Special Note: this will only sync data visible to everyone, any fields or rows hidden behind authentication will be missed. + +### 2) Generate test data + +Alternatively a faster way to get data into your instance is to generate a set of test data. + +The data generated has no relation to real world data, but should be good enough for testing purposes + +```sh +./manage.py pdb_generate_test_data --commit +``` + +## Admin + +The admin area can be accessed from the `/cp` endpoint, you will need to have a superuser to access it + +```sh +./manage.py createsuperuser +``` + ## Hangups ### Authentication not working diff --git a/peeringdb_server/management/commands/pdb_load_data.py b/peeringdb_server/management/commands/pdb_load_data.py new file mode 100644 index 00000000..9c3c5f30 --- /dev/null +++ b/peeringdb_server/management/commands/pdb_load_data.py @@ -0,0 +1,110 @@ +import logging +import datetime + +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.core.exceptions import ValidationError, ObjectDoesNotExist +from django.db.models.signals import post_save, pre_delete, pre_save +from django.conf import settings + +from peeringdb_server import models as pdb_models +from peeringdb_server import signals +from django_peeringdb import models as djpdb_models +from django_peeringdb import sync + +def sync_obj(cls, row): + """ + we need to override django peeringdb's sync_obj function + because our models differ slightly, not pretty, but good enough + for now. + """ + + if row.get("status") != "ok": + return + + try: + obj = cls.objects.get(pk=row['id']) + + except cls.DoesNotExist: + obj = cls() + + for k, v in row.items(): + if k in ["latitude", "longitude"] and v: + v = "{:3.6f}".format(float(v)) + elif k == "fac_id": + k = "facility_id" + elif k == "net_id": + k = "network_id" + elif k == "info_prefixes4": + v = max(v, 50000) + elif k == "info_prefixes6": + v = max(v, 10000) + try: + setattr(obj, k, v) + except AttributeError: + pass + + print(obj, obj.id) + + try: + obj.full_clean() + except ValidationError as e: + pass + + + for field in cls._meta.get_fields(): + ftyp = cls._meta.get_field(field.name) + value = getattr(obj, field.name, None) + if isinstance(value, datetime.datetime): + setattr(obj, field.name, value.replace(tzinfo=None)) + else: + if hasattr(ftyp, "related_name") and ftyp.multiple: + continue + else: + try: + setattr(obj, field.name, value) + except AttributeError: + pass + + obj.save() + return + +sync.sync_obj = sync_obj + + +class Command(BaseCommand): + help = "Load initial data from another peeringdb instance" + + def add_arguments(self, parser): + parser.add_argument("--url", default="https://www.peeringdb.com/api/", type=str) + + def handle(self, *args, **options): + if settings.RELEASE_ENV != "dev": + raise Exception("This command can only be run on dev instances") + + + settings.USE_TZ = False + settings.PEERINGDB_SYNC_URL = options.get("url") + pre_save.disconnect(signals.addressmodel_save, sender=pdb_models.Facility) + + djpdb_models.all_models = [ + pdb_models.Organization, + pdb_models.Facility, + pdb_models.Network, + pdb_models.InternetExchange, + pdb_models.InternetExchangeFacility, + pdb_models.IXLan, + pdb_models.IXLanPrefix, + pdb_models.NetworkContact, + pdb_models.NetworkFacility, + pdb_models.NetworkIXLan + ] + + for model in djpdb_models.all_models: + count = model.objects.all().count() + if count > 0: + self.stdout.write("This command should only be called on an empty database") + self.stdout.write("We already found {} {} in the database, aborting.".format(count, model)) + return + + call_command("pdb_sync")