From c73b9c7f8c8e0b6263a01f3b51ee8c9bb670cb18 Mon Sep 17 00:00:00 2001 From: Hugo Levy-Falk Date: Fri, 3 Apr 2020 16:57:51 +0200 Subject: [PATCH] TTLs, requirements and re2O validation. --- .gitignore | 3 +- Pipfile | 15 +++++++++ Pipfile.lock | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 34 ++++++++++++++++++++ gandi.py | 23 ++++++++++---- logging.conf | 2 +- main.py | 69 +++++++++++++++++++++++++++------------- requirements.txt | 8 +++++ 8 files changed, 205 insertions(+), 31 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 330bc34..3be6a97 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ config.toml last_update_*.toml +last_update -Pipfile -*.lock # Created by https://www.gitignore.io/api/venv,dotenv,python # Edit at https://www.gitignore.io/?templates=venv,dotenv,python diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..e7dae2e --- /dev/null +++ b/Pipfile @@ -0,0 +1,15 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +toml = "*" +requests = "*" +iso8601 = "*" +click = "*" + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..2515bb5 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,82 @@ +{ + "_meta": { + "hash": { + "sha256": "4207cfaa155c2d4eed2793b4e2fdc75de677ae0ea656239b47ff1ab0b4407b60" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + ], + "version": "==2019.11.28" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", + "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + ], + "index": "pypi", + "version": "==7.1.1" + }, + "idna": { + "hashes": [ + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + ], + "version": "==2.9" + }, + "iso8601": { + "hashes": [ + "sha256:210e0134677cc0d02f6028087fee1df1e1d76d372ee1db0bf30bf66c5c1c89a3", + "sha256:49c4b20e1f38aa5cf109ddcd39647ac419f928512c869dc01d5c7098eddede82", + "sha256:bbbae5fb4a7abfe71d4688fd64bff70b91bbd74ef6a99d964bab18f7fdf286dd" + ], + "index": "pypi", + "version": "==0.1.12" + }, + "requests": { + "hashes": [ + "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", + "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" + ], + "index": "pypi", + "version": "==2.23.0" + }, + "toml": { + "hashes": [ + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + ], + "index": "pypi", + "version": "==0.10.0" + }, + "urllib3": { + "hashes": [ + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + ], + "version": "==1.25.8" + } + }, + "develop": {} +} diff --git a/README.md b/README.md index e69de29..34105f7 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,34 @@ +# Re2o Gandi DNS + +This is a small script that can be the interface between Re2o and Gandi for the DNS. +So far it manages: + +* A records +* AAAA records +* CNAME records +* Origin record + +more can be added but I did not need them. + +The script keeps track of the records it can manage in toml files in `last_updates`, so that it won't delete records that you add manually in Gandi interface if they don't exist in Re2o. + +## Usage + +``` +Usage: main.py [OPTIONS] + +Options: + --config-dir TEXT Configuration directory. + --dry-run / --complete Performs a dry run. + --keep / --update Update service status on Re2o. Won't update if it is + a dry-run. + + --help Show this message and exit. + +``` + +## Installation + +Clone the repository (e.g. in `/usr/local/dns`) the install the requirements (with pip for example : `pip install -r requirements.txt`) and it's done. + +You can also setup a cron to run this on a regular basis. diff --git a/gandi.py b/gandi.py index 5ab5fc3..05a1ebb 100644 --- a/gandi.py +++ b/gandi.py @@ -1,4 +1,5 @@ import logging +import json import requests @@ -25,7 +26,13 @@ class GandiAPIClient: def _request(self, method, url, headers={}, params={}, *args, **kwargs): self.logger.debug( - "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r", method, url, headers, params, args, kwargs + "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r", + method, + url, + headers, + params, + args, + kwargs, ) headers.update({"Authorization": "Apikey %s" % self.api_key}) @@ -265,6 +272,7 @@ class Record(APIElement): rrset_name="", rrset_type=None, rrset_values=None, + rrset_ttl=10800, **kwargs ): endpoint = self.ENDPOINT.format( @@ -274,6 +282,7 @@ class Record(APIElement): self.rrset_name = rrset_name self.rrset_type = rrset_type self.rrset_values = rrset_values + self.rrset_ttl = rrset_ttl if self.rrset_values is None or self.rrset_type is None: self.fetch() @@ -286,23 +295,24 @@ class Record(APIElement): "rrset_name": self.rrset_name, "rrset_type": self.rrset_type, "rrset_values": self.rrset_values, + "rrset_ttl": self.rrset_ttl, } def as_api(self): - return { - "rrset_values": self.rrset_values - } + return {"rrset_values": self.rrset_values, "rrset_ttl": self.rrset_ttl} def from_dict(self, d): self.rrset_name = d["rrset_name"] self.rrset_type = d["rrset_type"] self.rrset_values = d["rrset_values"] + self.rrset_ttl = d.get("rrset_ttl", 10800) def __repr__(self): - return "" % ( + return "" % ( self.rrset_name, self.rrset_type, self.rrset_values, + self.rrset_ttl, ) def __eq__(self, other): @@ -310,6 +320,7 @@ class Record(APIElement): self.rrset_name == other.rrset_name and self.rrset_type == other.rrset_type and self.rrset_values == other.rrset_values + and self.rrset_ttl == other.rrset_ttl ) def __hash__(self): @@ -334,7 +345,7 @@ class DomainsRecords(APIElement): r.save() def from_dict(self, d): - # it's actually an array + # l is actually an array if isinstance(d, dict): l = d["records"] else: diff --git a/logging.conf b/logging.conf index 4f36520..37360b5 100644 --- a/logging.conf +++ b/logging.conf @@ -17,7 +17,7 @@ handlers= qualname=gandiAPI [logger_dns] -level=DEBUG +level=INFO handlers= qualname=dns diff --git a/main.py b/main.py index 8753e60..0dd349c 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import traceback import click import toml +import requests from re2oapi import Re2oAPIClient from gandi import GandiAPIClient, DomainsRecords, Record @@ -12,16 +13,17 @@ from gandi import GandiAPIClient, DomainsRecords, Record RUN_PATH = pathlib.Path(__file__).parent -def process_zone(zone, domains_records, logger): - pass - - @click.command() @click.option( "--config-dir", default=RUN_PATH.resolve(), help="Configuration directory." ) @click.option("--dry-run/--complete", default=False, help="Performs a dry run.") -def main(config_dir, dry_run): +@click.option( + "--keep/--update", + default=False, + help="Update service status on Re2o. Won't update if it is a dry-run.", +) +def main(config_dir, dry_run, keep): logging.config.fileConfig(config_dir / "logging.conf") logger = logging.getLogger("dns") logger.debug("Fetching configuration from %s.", config_dir) @@ -33,6 +35,9 @@ def main(config_dir, dry_run): use_tls=config["Re2o"]["use_TLS"], ) zones = re2o_client.list("dns/zones") + default_ttl = re2o_client.view("preferences/optionalmachine").get( + "default_dns_ttl", 10800 + ) default_API_key = config["Gandi"]["API_KEY"] @@ -40,6 +45,7 @@ def main(config_dir, dry_run): # Re2o has zones names begining with '.'. It is a bit difficult to translate # that into toml name = zone["name"][1:] + logger.debug(zone) try: configured_zone = config["Gandi"]["zone"][name] except KeyError as e: @@ -50,14 +56,14 @@ def main(config_dir, dry_run): gandi_client = GandiAPIClient(key) logger.info("Fetching last update for zone %s.", name) - last_update_file = config_dir / "last_update_{}.toml".format(name) + last_update_file = config_dir / "last_update" / "last_update_{}.toml".format(name) last_update_file.touch(mode=0o644, exist_ok=True) last_update = DomainsRecords(gandi_client, name, fetch=False) try: last_update.from_dict(toml.load(last_update_file)) except Exception as e: - logger.warning("Could not retrieve last update.") - logger.debug(e) + logger.warning("Could not retrieve last update.") + logger.debug(e) logger.info("Fetching current records for zone %s.", name) current_records = DomainsRecords(gandi_client, name) logger.info("Fetching re2o records for zone %s.", name) @@ -71,6 +77,7 @@ def main(config_dir, dry_run): rrset_name="@", rrset_type=Record.Types.A, rrset_values=[zone["originv4"]["ipv4"]], + rrset_ttl=zone["soa"].get("ttl", None) or default_ttl, ) ) @@ -82,41 +89,44 @@ def main(config_dir, dry_run): rrset_name="@", rrset_type=Record.Types.AAAA, rrset_values=[zone["originv6"]], + rrset_ttl=zone["soa"].get("ttl", None) or default_ttl, ) ) - for a_record in zone["a_records"]: + for record in zone["a_records"]: new_records.add( Record( gandi_client, name, - rrset_name=a_record["hostname"], + rrset_name=record["hostname"], rrset_type=Record.Types.A, - rrset_values=[a_record["ipv4"]], + rrset_values=[record["ipv4"]], + rrset_ttl=record.get("ttl", None) or default_ttl, ) ) - for aaaa_record in zone["aaaa_records"]: - logger.debug("aaaa records %r", aaaa_record) + for record in zone["aaaa_records"]: new_records.add( Record( gandi_client, name, - rrset_name=aaaa_record["hostname"], + rrset_name=record["hostname"], rrset_type=Record.Types.AAAA, - rrset_values=[ipv6["ipv6"] for ipv6 in aaaa_record["ipv6"]], + rrset_values=[ipv6["ipv6"] for ipv6 in record["ipv6"]], + rrset_ttl=record.get("ttl", None) or default_ttl, ) ) - for cname_record in zone["cname_records"]: + for record in zone["cname_records"]: new_records.add( Record( gandi_client, name, - rrset_name=cname_record["hostname"], + rrset_name=record["hostname"], rrset_type=Record.Types.CNAME, # The dot is to conform with Gandi API - rrset_values=[cname_record["alias"]+'.'], + rrset_values=[record["alias"] + "."], + rrset_ttl=record.get("ttl", None) or default_ttl, ) ) @@ -131,30 +141,45 @@ def main(config_dir, dry_run): logger.debug("I will delete : %r", to_be_deleted) if not dry_run: - saved = [] + saved = set() for r in to_be_deleted: logger.info("Deleting record %r for zone %s.", r, name) try: r.delete() except requests.exceptions.HTTPError as e: logger.error("Failed to delete %r for zone %s: %s", r, name, e) - saved.append(r) + saved.add(r) for r in to_be_added: logger.info("Adding record %r for zone %s.", r, name) try: r.save() - saved.append(r) + saved.add(r) except requests.exceptions.HTTPError as e: logger.error("Failed to add %r for zone %s: %s", r, name, e) logger.debug("Saving update for zone %s.", name) + + # the new last_update file should contain the old one, plus the ones registered on re2o, minus the ones that should have been removed and were not saved + managed = (set(last_update.records) | set(new_records)) - ( + to_be_deleted - saved + ) with last_update_file.open("w") as f: - toml.dump({"records": [r.as_dict() for r in saved]}, f) + toml.dump({"records": [r.as_dict() for r in managed]}, f) + else: logger.info("This is a dry run for zone %s.", name) logger.info("Records to be deleted : %r", to_be_deleted) logger.info("Records to be added : %r", to_be_added) + if not keep and not dry_run: + for service in re2o_client.list("services/regen/"): + if ( + service["hostname"] == client_hostname + and service["service_name"] == "dns" + and service["need_regen"] + ): + re2o_client.patch(service["api_url"], data={"need_regen": False}) + if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3485359 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +certifi==2019.11.28 +chardet==3.0.4 +click==7.1.1 +idna==2.9 +iso8601==0.1.12 +requests==2.23.0 +toml==0.10.0 +urllib3==1.25.8