TTLs, requirements and re2O validation.

.gitignore

@@ -1,8 +1,7 @@
 config.toml
 last_update_*.toml
+last_update
 
-Pipfile
-*.lock
 # Created by https://www.gitignore.io/api/venv,dotenv,python
 # Edit at https://www.gitignore.io/?templates=venv,dotenv,python
 

Pipfile

@@ -0,0 +1,15 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+toml = "*"
+requests = "*"
+iso8601 = "*"
+click = "*"
+
+[requires]
+python_version = "3.6"

Pipfile.lock

@@ -0,0 +1,82 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "4207cfaa155c2d4eed2793b4e2fdc75de677ae0ea656239b47ff1ab0b4407b60"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.6"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+            ],
+            "version": "==2019.11.28"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "click": {
+            "hashes": [
+                "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc",
+                "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"
+            ],
+            "index": "pypi",
+            "version": "==7.1.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
+                "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
+            ],
+            "version": "==2.9"
+        },
+        "iso8601": {
+            "hashes": [
+                "sha256:210e0134677cc0d02f6028087fee1df1e1d76d372ee1db0bf30bf66c5c1c89a3",
+                "sha256:49c4b20e1f38aa5cf109ddcd39647ac419f928512c869dc01d5c7098eddede82",
+                "sha256:bbbae5fb4a7abfe71d4688fd64bff70b91bbd74ef6a99d964bab18f7fdf286dd"
+            ],
+            "index": "pypi",
+            "version": "==0.1.12"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
+                "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"
+            ],
+            "index": "pypi",
+            "version": "==2.23.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+            ],
+            "index": "pypi",
+            "version": "==0.10.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
+            ],
+            "version": "==1.25.8"
+        }
+    },
+    "develop": {}
+}

README.md

@@ -0,0 +1,34 @@
+# Re2o Gandi DNS
+
+This is a small script that can be the interface between Re2o and Gandi for the DNS.
+So far it manages:
+
+* A records
+* AAAA records
+* CNAME records
+* Origin record
+
+more can be added but I did not need them.
+
+The script keeps track of the records it can manage in toml files in `last_updates`, so that it won't delete records that you add manually in Gandi interface if they don't exist in Re2o.
+
+## Usage
+
+```
+Usage: main.py [OPTIONS]
+
+Options:
+  --config-dir TEXT       Configuration directory.
+  --dry-run / --complete  Performs a dry run.
+  --keep / --update       Update service status on Re2o. Won't update if it is
+                          a dry-run.
+
+  --help                  Show this message and exit.
+
+```
+
+## Installation
+
+Clone the repository (e.g. in `/usr/local/dns`) the install the requirements (with pip for example : `pip install -r requirements.txt`) and it's done.
+
+You can also setup a cron to run this on a regular basis. 

gandi.py

@@ -1,4 +1,5 @@
 import logging
+import json
 
 import requests
 
@@ -25,7 +26,13 @@ class GandiAPIClient:
 
     def _request(self, method, url, headers={}, params={}, *args, **kwargs):
         self.logger.debug(
-            "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r", method, url, headers, params, args, kwargs
+            "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r",
+            method,
+            url,
+            headers,
+            params,
+            args,
+            kwargs,
         )
 
         headers.update({"Authorization": "Apikey  %s" % self.api_key})
@@ -265,6 +272,7 @@ class Record(APIElement):
         rrset_name="",
         rrset_type=None,
         rrset_values=None,
+        rrset_ttl=10800,
         **kwargs
     ):
         endpoint = self.ENDPOINT.format(
@@ -274,6 +282,7 @@ class Record(APIElement):
         self.rrset_name = rrset_name
         self.rrset_type = rrset_type
         self.rrset_values = rrset_values
+        self.rrset_ttl = rrset_ttl
         if self.rrset_values is None or self.rrset_type is None:
             self.fetch()
 
@@ -286,23 +295,24 @@ class Record(APIElement):
             "rrset_name": self.rrset_name,
             "rrset_type": self.rrset_type,
             "rrset_values": self.rrset_values,
+            "rrset_ttl": self.rrset_ttl,
         }
 
     def as_api(self):
-        return {
+        return {"rrset_values": self.rrset_values, "rrset_ttl": self.rrset_ttl}
-            "rrset_values": self.rrset_values
-        }
 
     def from_dict(self, d):
         self.rrset_name = d["rrset_name"]
         self.rrset_type = d["rrset_type"]
         self.rrset_values = d["rrset_values"]
+        self.rrset_ttl = d.get("rrset_ttl", 10800)
 
     def __repr__(self):
-        return "<Record name=%s, type=%s, values=%r>" % (
+        return "<Record name=%s, type=%s, values=%r, ttl=%s>" % (
             self.rrset_name,
             self.rrset_type,
             self.rrset_values,
+            self.rrset_ttl,
         )
 
     def __eq__(self, other):
@@ -310,6 +320,7 @@ class Record(APIElement):
             self.rrset_name == other.rrset_name
             and self.rrset_type == other.rrset_type
             and self.rrset_values == other.rrset_values
+            and self.rrset_ttl == other.rrset_ttl
         )
 
     def __hash__(self):
@@ -334,7 +345,7 @@ class DomainsRecords(APIElement):
             r.save()
 
     def from_dict(self, d):
-        # it's actually an array
+        # l is actually an array
         if isinstance(d, dict):
             l = d["records"]
         else:

logging.conf

@@ -17,7 +17,7 @@ handlers=
 qualname=gandiAPI
 
 [logger_dns]
-level=DEBUG
+level=INFO
 handlers=
 qualname=dns
 

main.py

@@ -5,6 +5,7 @@ import traceback
 
 import click
 import toml
+import requests
 
 from re2oapi import Re2oAPIClient
 from gandi import GandiAPIClient, DomainsRecords, Record
@@ -12,16 +13,17 @@ from gandi import GandiAPIClient, DomainsRecords, Record
 RUN_PATH = pathlib.Path(__file__).parent
 
 
-def process_zone(zone, domains_records, logger):
-    pass
-
-
 @click.command()
 @click.option(
     "--config-dir", default=RUN_PATH.resolve(), help="Configuration directory."
 )
 @click.option("--dry-run/--complete", default=False, help="Performs a dry run.")
-def main(config_dir, dry_run):
+@click.option(
+    "--keep/--update",
+    default=False,
+    help="Update service status on Re2o. Won't update if it is a dry-run.",
+)
+def main(config_dir, dry_run, keep):
     logging.config.fileConfig(config_dir / "logging.conf")
     logger = logging.getLogger("dns")
     logger.debug("Fetching configuration from %s.", config_dir)
@@ -33,6 +35,9 @@ def main(config_dir, dry_run):
         use_tls=config["Re2o"]["use_TLS"],
     )
     zones = re2o_client.list("dns/zones")
+    default_ttl = re2o_client.view("preferences/optionalmachine").get(
+        "default_dns_ttl", 10800
+    )
 
     default_API_key = config["Gandi"]["API_KEY"]
 
@@ -40,6 +45,7 @@ def main(config_dir, dry_run):
         # Re2o has zones names begining with '.'. It is a bit difficult to translate
         # that into toml
         name = zone["name"][1:]
+        logger.debug(zone)
         try:
             configured_zone = config["Gandi"]["zone"][name]
         except KeyError as e:
@@ -50,7 +56,7 @@ def main(config_dir, dry_run):
         gandi_client = GandiAPIClient(key)
 
         logger.info("Fetching last update for zone %s.", name)
-        last_update_file = config_dir / "last_update_{}.toml".format(name)
+        last_update_file = config_dir / "last_update" / "last_update_{}.toml".format(name)
         last_update_file.touch(mode=0o644, exist_ok=True)
         last_update = DomainsRecords(gandi_client, name, fetch=False)
         try:
@@ -71,6 +77,7 @@ def main(config_dir, dry_run):
                     rrset_name="@",
                     rrset_type=Record.Types.A,
                     rrset_values=[zone["originv4"]["ipv4"]],
+                    rrset_ttl=zone["soa"].get("ttl", None) or default_ttl,
                 )
             )
 
@@ -82,41 +89,44 @@ def main(config_dir, dry_run):
                     rrset_name="@",
                     rrset_type=Record.Types.AAAA,
                     rrset_values=[zone["originv6"]],
+                    rrset_ttl=zone["soa"].get("ttl", None) or default_ttl,
                 )
             )
 
-        for a_record in zone["a_records"]:
+        for record in zone["a_records"]:
             new_records.add(
                 Record(
                     gandi_client,
                     name,
-                    rrset_name=a_record["hostname"],
+                    rrset_name=record["hostname"],
                     rrset_type=Record.Types.A,
-                    rrset_values=[a_record["ipv4"]],
+                    rrset_values=[record["ipv4"]],
+                    rrset_ttl=record.get("ttl", None) or default_ttl,
                 )
             )
 
-        for aaaa_record in zone["aaaa_records"]:
+        for record in zone["aaaa_records"]:
-            logger.debug("aaaa records %r", aaaa_record)
             new_records.add(
                 Record(
                     gandi_client,
                     name,
-                    rrset_name=aaaa_record["hostname"],
+                    rrset_name=record["hostname"],
                     rrset_type=Record.Types.AAAA,
-                    rrset_values=[ipv6["ipv6"] for ipv6 in aaaa_record["ipv6"]],
+                    rrset_values=[ipv6["ipv6"] for ipv6 in record["ipv6"]],
+                    rrset_ttl=record.get("ttl", None) or default_ttl,
                 )
             )
 
-        for cname_record in zone["cname_records"]:
+        for record in zone["cname_records"]:
             new_records.add(
                 Record(
                     gandi_client,
                     name,
-                    rrset_name=cname_record["hostname"],
+                    rrset_name=record["hostname"],
                     rrset_type=Record.Types.CNAME,
                     # The dot is to conform with Gandi API
-                    rrset_values=[cname_record["alias"]+'.'],
+                    rrset_values=[record["alias"] + "."],
+                    rrset_ttl=record.get("ttl", None) or default_ttl,
                 )
             )
 
@@ -131,30 +141,45 @@ def main(config_dir, dry_run):
         logger.debug("I will delete : %r", to_be_deleted)
 
         if not dry_run:
-            saved = []
+            saved = set()
             for r in to_be_deleted:
                 logger.info("Deleting record %r for zone %s.", r, name)
                 try:
                     r.delete()
                 except requests.exceptions.HTTPError as e:
                     logger.error("Failed to delete %r for zone %s: %s", r, name, e)
-                    saved.append(r)
+                    saved.add(r)
 
             for r in to_be_added:
                 logger.info("Adding record %r for zone %s.", r, name)
                 try:
                     r.save()
-                    saved.append(r)
+                    saved.add(r)
                 except requests.exceptions.HTTPError as e:
                     logger.error("Failed to add %r for zone %s: %s", r, name, e)
             logger.debug("Saving update for zone %s.", name)
+
+            # the new last_update file should contain the old one, plus the ones registered on re2o, minus the ones that should have been removed and were not saved
+            managed = (set(last_update.records) | set(new_records)) - (
+                to_be_deleted - saved
+            )
             with last_update_file.open("w") as f:
-                toml.dump({"records": [r.as_dict() for r in saved]}, f)
+                toml.dump({"records": [r.as_dict() for r in managed]}, f)
+
         else:
             logger.info("This is a dry run for zone %s.", name)
             logger.info("Records to be deleted : %r", to_be_deleted)
             logger.info("Records to be added : %r", to_be_added)
 
+    if not keep and not dry_run:
+        for service in re2o_client.list("services/regen/"):
+            if (
+                service["hostname"] == client_hostname
+                and service["service_name"] == "dns"
+                and service["need_regen"]
+            ):
+                re2o_client.patch(service["api_url"], data={"need_regen": False})
+
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -0,0 +1,8 @@
+certifi==2019.11.28
+chardet==3.0.4
+click==7.1.1
+idna==2.9
+iso8601==0.1.12
+requests==2.23.0
+toml==0.10.0
+urllib3==1.25.8