Merge branch 'master' of git.20c.com:pdb/gh-peeringdb

config/facsimile/tmpl/_ALL_/init.sql

@@ -2,18 +2,18 @@
 {% for k, each in module.iteritems() %}
   {% if each.db %}
     {% if each.db.name %}
-create database if not exists {{env.rc.db.prefix}}{{each.db.name}} character set = utf8;
-grant all on {{env.rc.db.prefix}}{{each.db.name}}.* to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
-grant all on {{env.rc.db.prefix}}{{each.db.name}}.* to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
+create database if not exists {{env.rc.db.default.prefix}}{{each.db.name}} character set = utf8;
+grant all on {{env.rc.db.default.prefix}}{{each.db.name}}.* to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
+grant all on {{env.rc.db.default.prefix}}{{each.db.name}}.* to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
     {% endif %}
 
   {% for table in each.db.selectable %}
-grant select on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
-grant select on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
+grant select on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
+grant select on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
   {% endfor %}
   {% for table in each.db.writable %}
-grant all on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
-grant all on {{table}} to '{{env.rc.db.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
+grant all on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'localhost' identified by '{{each.password}}';
+grant all on {{table}} to '{{env.rc.db.default.prefix}}{{each.name}}'@'%.%.int' identified by '{{each.password}}';
   {% endfor %}
 
   {% endif %}

docs/deploy.md

@@ -9,16 +9,6 @@ This document uses the following variables
 export PDB_REPO=git@github.com:peeringdb/peeringdb.git
 ```
 
-### Install obfuscation tools (Only needed if you want to obfuscate js)
-
-```sh
-git clone src.20c.com:20c/sys-deploy
-mkdir -p ~/.local/google
-wget https://dl.google.com/closure-compiler/compiler-latest.zip
-unzip compiler-latest.zip
-mv compiler.jar ~/.local/google
-```
-
 ### Install facsimile
 
 ```sh
@@ -109,6 +99,45 @@ mysql -u root -p < .facsimile/tmp/RELEASE/dev/peeringdb/init.sql
 ./manage.py runserver
 ```
 
+## Populating with data
+
+There are currently 2 ways to get some data into your developer instance
+
+1) Sync from production peeringdb - slow, but accurate data
+2) Generate test data - fast, but marginally useful test data
+
+### 1) Sync from Production
+
+You can populate your data from peeringdb.com using
+
+```sh
+./manage.py pdb_load_data
+```
+
+However be prepared for this to take 15-20 minutes as it will not only sync the entities, but also set up usergroups for each organization and so forth.
+
+This can only be used to populate initial data. Once you have started adding / updating objects and your data diverges from production data, it is no longer useful to call this command nor will it allow you to.
+
+Special Note: this will only sync data visible to everyone, any fields or rows hidden behind authentication will be missed.
+
+### 2) Generate test data
+
+Alternatively a faster way to get data into your instance is to generate a set of test data.
+
+The data generated has no relation to real world data, but should be good enough for testing purposes
+
+```sh
+./manage.py pdb_generate_test_data --commit
+```
+
+## Admin
+
+The admin area can be accessed from the `/cp` endpoint, you will need to have a superuser to access it
+
+```sh
+./manage.py createsuperuser
+```
+
 ## Hangups
 
 ### Authentication not working

peeringdb_server/management/commands/pdb_load_data.py

@@ -0,0 +1,110 @@
+import logging
+import datetime
+
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+from django.core.exceptions import ValidationError, ObjectDoesNotExist
+from django.db.models.signals import post_save, pre_delete, pre_save
+from django.conf import settings
+
+from peeringdb_server import models as pdb_models
+from peeringdb_server import signals
+from django_peeringdb import models as djpdb_models
+from django_peeringdb import sync
+
+def sync_obj(cls, row):
+    """
+    we need to override django peeringdb's sync_obj function
+    because our models differ slightly, not pretty, but good enough
+    for now.
+    """
+
+    if row.get("status") != "ok":
+        return
+
+    try:
+        obj = cls.objects.get(pk=row['id'])
+
+    except cls.DoesNotExist:
+        obj = cls()
+
+    for k, v in row.items():
+        if k in ["latitude", "longitude"] and v:
+            v = "{:3.6f}".format(float(v))
+        elif k == "fac_id":
+            k = "facility_id"
+        elif k == "net_id":
+            k = "network_id"
+        elif k == "info_prefixes4":
+            v = max(v, 50000)
+        elif k == "info_prefixes6":
+            v = max(v, 10000)
+        try:
+            setattr(obj, k, v)
+        except AttributeError:
+            pass
+
+    print(obj, obj.id)
+
+    try:
+        obj.full_clean()
+    except ValidationError as e:
+        pass
+
+
+    for field in cls._meta.get_fields():
+        ftyp = cls._meta.get_field(field.name)
+        value = getattr(obj, field.name, None)
+        if isinstance(value, datetime.datetime):
+            setattr(obj, field.name, value.replace(tzinfo=None))
+        else:
+            if hasattr(ftyp, "related_name") and ftyp.multiple:
+                continue
+            else:
+                try:
+                    setattr(obj, field.name, value)
+                except AttributeError:
+                    pass
+
+    obj.save()
+    return
+
+sync.sync_obj = sync_obj
+
+
+class Command(BaseCommand):
+    help = "Load initial data from another peeringdb instance"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--url", default="https://www.peeringdb.com/api/", type=str)
+
+    def handle(self, *args, **options):
+        if settings.RELEASE_ENV != "dev":
+            raise Exception("This command can only be run on dev instances")
+
+
+        settings.USE_TZ = False
+        settings.PEERINGDB_SYNC_URL = options.get("url")
+        pre_save.disconnect(signals.addressmodel_save, sender=pdb_models.Facility)
+
+        djpdb_models.all_models = [
+            pdb_models.Organization,
+            pdb_models.Facility,
+            pdb_models.Network,
+            pdb_models.InternetExchange,
+            pdb_models.InternetExchangeFacility,
+            pdb_models.IXLan,
+            pdb_models.IXLanPrefix,
+            pdb_models.NetworkContact,
+            pdb_models.NetworkFacility,
+            pdb_models.NetworkIXLan
+        ]
+
+        for model in djpdb_models.all_models:
+            count = model.objects.all().count()
+            if count > 0:
+                self.stdout.write("This command should only be called on an empty database")
+                self.stdout.write("We already found {} {} in the database, aborting.".format(count, model))
+                return
+
+        call_command("pdb_sync")