From 6353a70fbe91099f7b63c89e7d002ada1fc99651 Mon Sep 17 00:00:00 2001 From: Hugo Levy-Falk Date: Fri, 3 Apr 2020 15:08:38 +0200 Subject: [PATCH] First basic functionnal version --- .gitignore | 129 ++++++++++++++++ .gitmodules | 3 + README.md | 0 config.example.toml | 16 ++ gandi.py | 349 ++++++++++++++++++++++++++++++++++++++++++++ logging.conf | 37 +++++ main.py | 160 ++++++++++++++++++++ re2oapi | 1 + 8 files changed, 695 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 README.md create mode 100644 config.example.toml create mode 100644 gandi.py create mode 100644 logging.conf create mode 100644 main.py create mode 160000 re2oapi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..330bc34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +config.toml +last_update_*.toml + +Pipfile +*.lock +# Created by https://www.gitignore.io/api/venv,dotenv,python +# Edit at https://www.gitignore.io/?templates=venv,dotenv,python + +### dotenv ### +.env + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### venv ### +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +pyvenv.cfg +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pip-selfcheck.json + +# End of https://www.gitignore.io/api/venv,dotenv,python diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3439e4e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "re2oapi"] + path = re2oapi + url = https://gitlab.federez.net/re2o/re2oapi diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..4f6e0be --- /dev/null +++ b/config.example.toml @@ -0,0 +1,16 @@ +[Re2o] +hostname = "re2o.example.net" +username = "server" +password = "plopiplop" +use_TLS = false +[Gandi] +# Default API key +API_KEY = "thisissecret" + +# Zones are specified like this. names shouldn't begin with '.' as in re2o. +[Gandi.zone."some.domain.tld"] +# uses the default API_KEY + +[Gandi.zone."another.domain.tld"] +# uses its own API_KEY +API_KEY = "THIS IS SECRET" diff --git a/gandi.py b/gandi.py new file mode 100644 index 0000000..5ab5fc3 --- /dev/null +++ b/gandi.py @@ -0,0 +1,349 @@ +import logging + +import requests + +API_URL = "https://api.gandi.net/v5/" + + +class GandiAPIClient: + """Wrapper for the Gandi API client. + + It handles Gandi API over https for LiveDNS. Logging is done through the + `gandiAPI` logger. See `logging` package documentation for how to configure + it. + """ + + def __init__(self, api_key): + """Initializes the API client. + + Args: + api_key: the API key as provided by Gandi. + log_level: logging level (see `logging` package documentation) + """ + self.logger = logging.getLogger("gandiAPI") + self.api_key = api_key + + def _request(self, method, url, headers={}, params={}, *args, **kwargs): + self.logger.debug( + "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r", method, url, headers, params, args, kwargs + ) + + headers.update({"Authorization": "Apikey %s" % self.api_key}) + response = getattr(requests, method)( + url, headers=headers, params=params, allow_redirects=False, *args, **kwargs + ) + + self.logger.debug("Got response %r.", response) + response.raise_for_status() + try: + ret = response.json() + except json.decoder.JSONDecodeError as e: + ret = None + self.logger.debug("No valid JSON in the response.") + self.logger.debug("Request {} {} successful.".format(method, response.url)) + return ret + + def delete(self, *args, **kwargs): + """Performs a DELETE request. + + DELETE request on a given URL that acts like `requests.delete` except + that authentication to the API is automatically done and JSON response + is decoded. + + Args: + url: The URL of the requests. + *args: See `requests.delete` parameters. + **kwargs: See `requests.delete` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("delete", *args, **kwargs) + + def get(self, *args, **kwargs): + """Performs a GET request. + + GET request on a given URL that acts like `requests.get` except that + authentication to the API is automatically done and JSON response is + decoded. + + Args: + url: The URL of the requests. + *args: See `requests.get` parameters. + **kwargs: See `requests.get` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("get", *args, **kwargs) + + def head(self, *args, **kwargs): + """Performs a HEAD request. + + HEAD request on a given URL that acts like `requests.head` except that + authentication to the API is automatically done and JSON response is + decoded. + + Args: + url: The URL of the requests. + *args: See `requests.head` parameters. + **kwargs: See `requests.head` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("get", *args, **kwargs) + + def option(self, *args, **kwargs): + """Performs a OPTION request. + + OPTION request on a given URL that acts like `requests.option` except + that authentication to the API is automatically done and JSON response + is decoded. + + Args: + url: The URL of the requests. + *args: See `requests.option` parameters. + **kwargs: See `requests.option` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("get", *args, **kwargs) + + def patch(self, *args, **kwargs): + """Performs a PATCH request. + + PATCH request on a given URL that acts like `requests.patch` except + that authentication to the API is automatically done and JSON response + is decoded. + + Args: + url: The URL of the requests. + *args: See `requests.patch` parameters. + **kwargs: See `requests.patch` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("patch", *args, **kwargs) + + def post(self, *args, **kwargs): + """Performs a POST request. + + POST request on a given URL that acts like `requests.post` except that + authentication to the API is automatically done and JSON response is + decoded. + + Args: + url: The URL of the requests. + *args: See `requests.post` parameters. + **kwargs: See `requests.post` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("post", *args, **kwargs) + + def put(self, *args, **kwargs): + """Performs a PUT request. + + PUT request on a given URL that acts like `requests.put` except that + authentication to the API is automatically done and JSON response is + decoded. + + Args: + url: The URL of the requests. + *args: See `requests.put` parameters. + **kwargs: See `requests.put` parameters. + + Returns: + The JSON-decoded result of the request. + + Raises: + requests.exceptions.RequestException: An error occured while + performing the request. + exceptions.PermissionDenied: The user does not have the right + to perform this request. + """ + return self._request("put", *args, **kwargs) + + +class APIElement: + def __init__(self, client, api_endpoint): + self.client = client + self.api_endpoint = api_endpoint + + def save(self): + data = self.as_api() + if self.exists: + self.client.put(API_URL + self.api_endpoint, json=data) + else: + self.client.post(API_URL + self.api_endpoint, json=data) + + @property + def exists(self): + try: + r = self.client.get(API_URL + self.api_endpoint) + return True + except requests.exceptions.HTTPError: + return False + + def fetch(self): + r = self.client.get(API_URL + self.api_endpoint) + self.from_dict(r) + + def delete(self): + self.client.delete(API_URL + self.api_endpoint) + + +class Record(APIElement): + class Types: + A = "A" + AAAA = "AAAA" + ALIAS = "ALIAS" + CAA = "CAA" + CDS = "CDS" + CNAME = "CNAME" + DNAME = "DNAME" + DS = "DS" + KEY = "KEY" + LOC = "LOC" + MX = "MX" + NS = "NS" + OPENPGPKEY = "OPENPGPKEY" + PTR = "PTR" + SPF = "SPF" + SRV = "SRV" + SSHFP = "SSHFP" + TLSA = "TLSA" + TXT = "TXT" + WKS = "WKS" + + ENDPOINT = "livedns/domains/{fqdn}/records/{rrset_name}/{rrset_type}" + + def __init__( + self, + client, + domain, + rrset_name="", + rrset_type=None, + rrset_values=None, + **kwargs + ): + endpoint = self.ENDPOINT.format( + fqdn=domain, rrset_name=rrset_name, rrset_type=rrset_type + ) + super().__init__(client, endpoint) + self.rrset_name = rrset_name + self.rrset_type = rrset_type + self.rrset_values = rrset_values + if self.rrset_values is None or self.rrset_type is None: + self.fetch() + + @classmethod + def from_name(cls, client, domain, name): + cls(client, domain, rrset_name=name) + + def as_dict(self): + return { + "rrset_name": self.rrset_name, + "rrset_type": self.rrset_type, + "rrset_values": self.rrset_values, + } + + def as_api(self): + return { + "rrset_values": self.rrset_values + } + + def from_dict(self, d): + self.rrset_name = d["rrset_name"] + self.rrset_type = d["rrset_type"] + self.rrset_values = d["rrset_values"] + + def __repr__(self): + return "" % ( + self.rrset_name, + self.rrset_type, + self.rrset_values, + ) + + def __eq__(self, other): + return ( + self.rrset_name == other.rrset_name + and self.rrset_type == other.rrset_type + and self.rrset_values == other.rrset_values + ) + + def __hash__(self): + return hash(repr(self)) + + +class DomainsRecords(APIElement): + ENDPOINT = "livedns/domains/{fqdn}/records" + + def __init__(self, client, fqdn, records=None, fetch=True): + endpoint = self.ENDPOINT.format(fqdn=fqdn) + super().__init__(client, endpoint) + self.fqdn = fqdn + self.records = records + if self.records is None and fetch: + self.fetch() + else: + self.records = [] + + def save(self): + for r in self.records: + r.save() + + def from_dict(self, d): + # it's actually an array + if isinstance(d, dict): + l = d["records"] + else: + l = d + logging.getLogger("gandiAPI").debug(l) + self.records = [Record(self.client, self.fqdn, **r) for r in l] + + def as_dict(self): + return [r.as_dict() for r in self.records] + + def __repr__(self): + return "" % (self.fqdn, self.records) diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..4f36520 --- /dev/null +++ b/logging.conf @@ -0,0 +1,37 @@ +[loggers] +keys=root,dns,gandiAPI + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[logger_gandiAPI] +level=INFO +handlers= +qualname=gandiAPI + +[logger_dns] +level=DEBUG +handlers= +qualname=dns + +[logger_re2oapi] +qualname=re2oapi.re2oapi.client +level=INFO +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/main.py b/main.py new file mode 100644 index 0000000..8753e60 --- /dev/null +++ b/main.py @@ -0,0 +1,160 @@ +import logging +import logging.config +import pathlib +import traceback + +import click +import toml + +from re2oapi import Re2oAPIClient +from gandi import GandiAPIClient, DomainsRecords, Record + +RUN_PATH = pathlib.Path(__file__).parent + + +def process_zone(zone, domains_records, logger): + pass + + +@click.command() +@click.option( + "--config-dir", default=RUN_PATH.resolve(), help="Configuration directory." +) +@click.option("--dry-run/--complete", default=False, help="Performs a dry run.") +def main(config_dir, dry_run): + logging.config.fileConfig(config_dir / "logging.conf") + logger = logging.getLogger("dns") + logger.debug("Fetching configuration from %s.", config_dir) + config = toml.load(config_dir / "config.toml") + re2o_client = Re2oAPIClient( + config["Re2o"]["hostname"], + config["Re2o"]["username"], + config["Re2o"]["password"], + use_tls=config["Re2o"]["use_TLS"], + ) + zones = re2o_client.list("dns/zones") + + default_API_key = config["Gandi"]["API_KEY"] + + for zone in zones: + # Re2o has zones names begining with '.'. It is a bit difficult to translate + # that into toml + name = zone["name"][1:] + try: + configured_zone = config["Gandi"]["zone"][name] + except KeyError as e: + logger.error("Could not find zone named %s in configuration.", e) + continue + + key = configured_zone.get("API_KEY", default_API_key) + gandi_client = GandiAPIClient(key) + + logger.info("Fetching last update for zone %s.", name) + last_update_file = config_dir / "last_update_{}.toml".format(name) + last_update_file.touch(mode=0o644, exist_ok=True) + last_update = DomainsRecords(gandi_client, name, fetch=False) + try: + last_update.from_dict(toml.load(last_update_file)) + except Exception as e: + logger.warning("Could not retrieve last update.") + logger.debug(e) + logger.info("Fetching current records for zone %s.", name) + current_records = DomainsRecords(gandi_client, name) + logger.info("Fetching re2o records for zone %s.", name) + new_records = set() + + if zone["originv4"]: + new_records.add( + Record( + gandi_client, + name, + rrset_name="@", + rrset_type=Record.Types.A, + rrset_values=[zone["originv4"]["ipv4"]], + ) + ) + + if zone["originv6"]: + new_records.add( + Record( + gandi_client, + name, + rrset_name="@", + rrset_type=Record.Types.AAAA, + rrset_values=[zone["originv6"]], + ) + ) + + for a_record in zone["a_records"]: + new_records.add( + Record( + gandi_client, + name, + rrset_name=a_record["hostname"], + rrset_type=Record.Types.A, + rrset_values=[a_record["ipv4"]], + ) + ) + + for aaaa_record in zone["aaaa_records"]: + logger.debug("aaaa records %r", aaaa_record) + new_records.add( + Record( + gandi_client, + name, + rrset_name=aaaa_record["hostname"], + rrset_type=Record.Types.AAAA, + rrset_values=[ipv6["ipv6"] for ipv6 in aaaa_record["ipv6"]], + ) + ) + + for cname_record in zone["cname_records"]: + new_records.add( + Record( + gandi_client, + name, + rrset_name=cname_record["hostname"], + rrset_type=Record.Types.CNAME, + # The dot is to conform with Gandi API + rrset_values=[cname_record["alias"]+'.'], + ) + ) + + # Delete records added by this script that are not in new_records. + to_be_deleted = set(last_update.records) & ( + set(current_records.records) - set(new_records) + ) + # Add records that are in new_records except if they are already there. + to_be_added = set(new_records) - set(current_records.records) + logger.debug("Re2o records are %r", new_records) + logger.debug("I will add : %r", to_be_added) + logger.debug("I will delete : %r", to_be_deleted) + + if not dry_run: + saved = [] + for r in to_be_deleted: + logger.info("Deleting record %r for zone %s.", r, name) + try: + r.delete() + except requests.exceptions.HTTPError as e: + logger.error("Failed to delete %r for zone %s: %s", r, name, e) + saved.append(r) + + for r in to_be_added: + logger.info("Adding record %r for zone %s.", r, name) + try: + r.save() + saved.append(r) + except requests.exceptions.HTTPError as e: + logger.error("Failed to add %r for zone %s: %s", r, name, e) + logger.debug("Saving update for zone %s.", name) + with last_update_file.open("w") as f: + toml.dump({"records": [r.as_dict() for r in saved]}, f) + else: + logger.info("This is a dry run for zone %s.", name) + logger.info("Records to be deleted : %r", to_be_deleted) + logger.info("Records to be added : %r", to_be_added) + + +if __name__ == "__main__": + main() diff --git a/re2oapi b/re2oapi new file mode 160000 index 0000000..ffaed92 --- /dev/null +++ b/re2oapi @@ -0,0 +1 @@ +Subproject commit ffaed921030deb6b6b01649709666807feb95370