Common libs

bird.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import re
+
+import pprint
+
+BIRD_REGEXP = r"(.*) \[(.*)] \*? ?\((.*)\) \[(.*)\]"
+
+def parse_bird(stream):
+    """Should take the output of "birdc 'show route primary'" as input.
+    Returns a dictionary with all the information."""
+    d = dict()
+    for l in stream.readlines()[1:]:
+        #l = l.translate(l.maketrans("", "",'()[]*'))
+        data = re.match(BIRD_REGEXP, l.strip())
+        try:
+            data = data.groups()
+            prefix = data[0].split()[0]
+            d[prefix] = {
+                "nexthop": data[0].split()[2],
+                "iface": data[0].split()[4],
+                "bgp_peer": data[1].split()[0],
+                "up_since": data[1].split()[1],
+                "localpref": int(data[2]),
+                "origin_as": int(data[3][2:-1]),
+                "last_updated": int(time.time()),
+                "current": True,
+            }
+        except Exception as e:
+            print("Unable to parse line '{}' : {}".format(l, e))
+    return d
+
+def update(db, new):
+    """Modify db in place, updating it to reflect the new data from bird."""
+    for prefix in db:
+        db[prefix]["current"] = False
+    for prefix in new:
+        new[prefix]["first_seen"] = db[prefix]["first_seen"] if prefix in db and "first_seen" in db[prefix] and isinstance(db[prefix]["first_seen"], int) else int(time.time())
+    db.update(new)
+
+if __name__ == '__main__':
+    pprint.pprint(parse_bird(sys.stdin))

registry.py

@@ -0,0 +1,149 @@
+"""Parsing library for the dn42 registry.
+
+All data is parsed as python dictionaries, and can be exported as JSON.
+Example usage:
+
+    from registry import Dns
+    dns = Dns("/home/example/net.dn42.registry")
+    print(dns.data["internal.dn42"])
+    with open("/tmp/dns.json", "w") as f:
+        dns.write_json(f)
+
+There are several classes available: Inetnum, AutNum, etc (see below).
+They follow the naming convention of the folders in data/, using
+CamelCase.
+
+There is also a big class containing all relevant data from the registry,
+in case you're not afraid of the performance penalty of parsing 1145 small
+files:
+
+    from registry import Registry
+    dn42 = Registry("/home/example/net.dn42.registry")
+    print(dn42.dns.data)
+    print(dn42.inetnum.data)
+
+"""
+
+import os
+import json
+
+def parse_record(stream):
+    """General parsing of the "key: value" syntax. Returns a key -> [values]
+    dictionary.
+    """
+    d = dict()
+    for entry in stream.readlines():
+        try:
+            key, value = [s.strip() for s in entry.split(':', 1)]
+            if not key in d:
+                d[key] = list()
+            d[key].append(value)
+        except ValueError: pass
+    return d
+
+
+def parse_records(records_dir, replace_underscore=None):
+    """Takes a directory containing records, and builds a dictionary mapping
+    the filename of each record to its parsed data.  If requested, we
+    transform '_' by [replace_underscore] in the name of the records.
+
+    """
+    records = dict()
+    for record in os.listdir(records_dir):
+        record_path = os.path.join(records_dir, record)
+        record_key = record.replace('_', replace_underscore) if replace_underscore else record
+        with open(record_path, "r") as f:
+            records[record_key] = parse_record(f)
+    return records
+
+
+class Dn42Entry(object):
+    """Should not be used directly, use one of the sub-classes below."""
+    directory = ""
+    data = dict()
+
+    def __init__(self, registrypath, replace_underscore=None):
+        fullpath = os.path.join(registrypath, "data", self.directory)
+        self.data = parse_records(fullpath, replace_underscore)
+
+    def write_json(self, stream):
+        json.dump(self.data, stream)
+
+    def get_json(self):
+        return json.dumps(self.data)
+
+
+class Dns(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "dns"
+        super(Dns, self).__init__(registrypath)
+
+
+class Inetnum(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "inetnum"
+        super(Inetnum, self).__init__(registrypath, '/')
+
+class Inet6num(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "inet6num"
+        super(Inet6num, self).__init__(registrypath, '/')
+
+
+class Route(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "route"
+        super(Route, self).__init__(registrypath, '/')
+
+class Route6(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "route6"
+        super(Route6, self).__init__(registrypath, '/')
+
+
+class Person(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "person"
+        super(Person, self).__init__(registrypath)
+
+class Organisation(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "organisation"
+        super(Organisation, self).__init__(registrypath)
+
+class Mntner(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "mntner"
+        super(Mntner, self).__init__(registrypath)
+
+
+class AsBlock(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "as-block"
+        super(AsBlock, self).__init__(registrypath, '-')
+
+class AsSet(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "as-set"
+        super(AsSet, self).__init__(registrypath)
+
+class AutNum(Dn42Entry):
+    def __init__(self, registrypath):
+        self.directory = "aut-num"
+        super(AutNum, self).__init__(registrypath)
+
+
+class Registry(Dn42Entry):
+    """Big class that provides all available data from the registry."""
+    def __init__(self, registrypath):
+        self.dns = Dns(registrypath)
+        self.inetnum = Inetnum(registrypath)
+        self.inet6num = Inet6num(registrypath)
+        self.route = Route(registrypath)
+        self.route6 = Route6(registrypath)
+        self.person = Person(registrypath)
+        self.organisation = Organisation(registrypath)
+        self.mntner = Mntner(registrypath)
+        self.asblock = AsBlock(registrypath)
+        self.asset = AsSet(registrypath)
+        self.autnum = AutNum(registrypath)

utils.py

@@ -0,0 +1,19 @@
+import json
+
+def read_json(filename):
+    try:
+        with open(filename, "r") as f:
+            data = json.load(f)
+    except FileNotFoundError as e:
+        print("Warning:", e)
+        print("Could not find json file {}, returning an empty dict".format(filename))
+        data = dict()
+    except ValueError as e:
+        print("Warning:", e)
+        print("Could not decode json data of file {}, returning an empty dict".format(filename))
+        data = dict()
+    return data
+
+def write_json(data, filename):
+    with open(filename, "w") as f:
+        json.dump(data, f)