diff --git a/Pipfile b/Pipfile index 532424f..d150a1a 100644 --- a/Pipfile +++ b/Pipfile @@ -10,5 +10,6 @@ toml = "*" requests = "*" iso8601 = "*" click = "*" +jinja2 = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index 5c29386..e3c341b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7584b395509a753bbc6c120c119fe1adf1425def6e6954116d2768fa204f8e25" + "sha256": "e14cd0676a0f2cccccf0b0f8094419a09a4a7dec2b38bc2ae3a23abff3ccaaa9" }, "pipfile-spec": 6, "requires": {}, @@ -52,6 +52,52 @@ "index": "pypi", "version": "==0.1.12" }, + "jinja2": { + "hashes": [ + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" + ], + "index": "pypi", + "version": "==2.11.2" + }, + "markupsafe": { + "hashes": [ + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" + ], + "version": "==1.1.1" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", diff --git a/README.md b/README.md index 34105f7..aeff8fd 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,4 @@ -# Re2o Gandi DNS +# Re2o Virtual Emails -This is a small script that can be the interface between Re2o and Gandi for the DNS. -So far it manages: +This is a small service for extracting emails from re2o. It is designed to create virtual mailboxes for users. There is already a [re2o service](https://gitlab.federez.net/re2o/mail-server/) for emails, but it is for local emails. I don't want that. -* A records -* AAAA records -* CNAME records -* Origin record - -more can be added but I did not need them. - -The script keeps track of the records it can manage in toml files in `last_updates`, so that it won't delete records that you add manually in Gandi interface if they don't exist in Re2o. - -## Usage - -``` -Usage: main.py [OPTIONS] - -Options: - --config-dir TEXT Configuration directory. - --dry-run / --complete Performs a dry run. - --keep / --update Update service status on Re2o. Won't update if it is - a dry-run. - - --help Show this message and exit. - -``` - -## Installation - -Clone the repository (e.g. in `/usr/local/dns`) the install the requirements (with pip for example : `pip install -r requirements.txt`) and it's done. - -You can also setup a cron to run this on a regular basis. diff --git a/config.example.toml b/config.example.toml index 4f6e0be..c430bb4 100644 --- a/config.example.toml +++ b/config.example.toml @@ -3,14 +3,3 @@ hostname = "re2o.example.net" username = "server" password = "plopiplop" use_TLS = false -[Gandi] -# Default API key -API_KEY = "thisissecret" - -# Zones are specified like this. names shouldn't begin with '.' as in re2o. -[Gandi.zone."some.domain.tld"] -# uses the default API_KEY - -[Gandi.zone."another.domain.tld"] -# uses its own API_KEY -API_KEY = "THIS IS SECRET" diff --git a/gandi.py b/gandi.py deleted file mode 100644 index 05a1ebb..0000000 --- a/gandi.py +++ /dev/null @@ -1,360 +0,0 @@ -import logging -import json - -import requests - -API_URL = "https://api.gandi.net/v5/" - - -class GandiAPIClient: - """Wrapper for the Gandi API client. - - It handles Gandi API over https for LiveDNS. Logging is done through the - `gandiAPI` logger. See `logging` package documentation for how to configure - it. - """ - - def __init__(self, api_key): - """Initializes the API client. - - Args: - api_key: the API key as provided by Gandi. - log_level: logging level (see `logging` package documentation) - """ - self.logger = logging.getLogger("gandiAPI") - self.api_key = api_key - - def _request(self, method, url, headers={}, params={}, *args, **kwargs): - self.logger.debug( - "Requesting %s: %s, headers=%r, params=%r, args=%r, kwargs=%r", - method, - url, - headers, - params, - args, - kwargs, - ) - - headers.update({"Authorization": "Apikey %s" % self.api_key}) - response = getattr(requests, method)( - url, headers=headers, params=params, allow_redirects=False, *args, **kwargs - ) - - self.logger.debug("Got response %r.", response) - response.raise_for_status() - try: - ret = response.json() - except json.decoder.JSONDecodeError as e: - ret = None - self.logger.debug("No valid JSON in the response.") - self.logger.debug("Request {} {} successful.".format(method, response.url)) - return ret - - def delete(self, *args, **kwargs): - """Performs a DELETE request. - - DELETE request on a given URL that acts like `requests.delete` except - that authentication to the API is automatically done and JSON response - is decoded. - - Args: - url: The URL of the requests. - *args: See `requests.delete` parameters. - **kwargs: See `requests.delete` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("delete", *args, **kwargs) - - def get(self, *args, **kwargs): - """Performs a GET request. - - GET request on a given URL that acts like `requests.get` except that - authentication to the API is automatically done and JSON response is - decoded. - - Args: - url: The URL of the requests. - *args: See `requests.get` parameters. - **kwargs: See `requests.get` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("get", *args, **kwargs) - - def head(self, *args, **kwargs): - """Performs a HEAD request. - - HEAD request on a given URL that acts like `requests.head` except that - authentication to the API is automatically done and JSON response is - decoded. - - Args: - url: The URL of the requests. - *args: See `requests.head` parameters. - **kwargs: See `requests.head` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("get", *args, **kwargs) - - def option(self, *args, **kwargs): - """Performs a OPTION request. - - OPTION request on a given URL that acts like `requests.option` except - that authentication to the API is automatically done and JSON response - is decoded. - - Args: - url: The URL of the requests. - *args: See `requests.option` parameters. - **kwargs: See `requests.option` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("get", *args, **kwargs) - - def patch(self, *args, **kwargs): - """Performs a PATCH request. - - PATCH request on a given URL that acts like `requests.patch` except - that authentication to the API is automatically done and JSON response - is decoded. - - Args: - url: The URL of the requests. - *args: See `requests.patch` parameters. - **kwargs: See `requests.patch` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("patch", *args, **kwargs) - - def post(self, *args, **kwargs): - """Performs a POST request. - - POST request on a given URL that acts like `requests.post` except that - authentication to the API is automatically done and JSON response is - decoded. - - Args: - url: The URL of the requests. - *args: See `requests.post` parameters. - **kwargs: See `requests.post` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("post", *args, **kwargs) - - def put(self, *args, **kwargs): - """Performs a PUT request. - - PUT request on a given URL that acts like `requests.put` except that - authentication to the API is automatically done and JSON response is - decoded. - - Args: - url: The URL of the requests. - *args: See `requests.put` parameters. - **kwargs: See `requests.put` parameters. - - Returns: - The JSON-decoded result of the request. - - Raises: - requests.exceptions.RequestException: An error occured while - performing the request. - exceptions.PermissionDenied: The user does not have the right - to perform this request. - """ - return self._request("put", *args, **kwargs) - - -class APIElement: - def __init__(self, client, api_endpoint): - self.client = client - self.api_endpoint = api_endpoint - - def save(self): - data = self.as_api() - if self.exists: - self.client.put(API_URL + self.api_endpoint, json=data) - else: - self.client.post(API_URL + self.api_endpoint, json=data) - - @property - def exists(self): - try: - r = self.client.get(API_URL + self.api_endpoint) - return True - except requests.exceptions.HTTPError: - return False - - def fetch(self): - r = self.client.get(API_URL + self.api_endpoint) - self.from_dict(r) - - def delete(self): - self.client.delete(API_URL + self.api_endpoint) - - -class Record(APIElement): - class Types: - A = "A" - AAAA = "AAAA" - ALIAS = "ALIAS" - CAA = "CAA" - CDS = "CDS" - CNAME = "CNAME" - DNAME = "DNAME" - DS = "DS" - KEY = "KEY" - LOC = "LOC" - MX = "MX" - NS = "NS" - OPENPGPKEY = "OPENPGPKEY" - PTR = "PTR" - SPF = "SPF" - SRV = "SRV" - SSHFP = "SSHFP" - TLSA = "TLSA" - TXT = "TXT" - WKS = "WKS" - - ENDPOINT = "livedns/domains/{fqdn}/records/{rrset_name}/{rrset_type}" - - def __init__( - self, - client, - domain, - rrset_name="", - rrset_type=None, - rrset_values=None, - rrset_ttl=10800, - **kwargs - ): - endpoint = self.ENDPOINT.format( - fqdn=domain, rrset_name=rrset_name, rrset_type=rrset_type - ) - super().__init__(client, endpoint) - self.rrset_name = rrset_name - self.rrset_type = rrset_type - self.rrset_values = rrset_values - self.rrset_ttl = rrset_ttl - if self.rrset_values is None or self.rrset_type is None: - self.fetch() - - @classmethod - def from_name(cls, client, domain, name): - cls(client, domain, rrset_name=name) - - def as_dict(self): - return { - "rrset_name": self.rrset_name, - "rrset_type": self.rrset_type, - "rrset_values": self.rrset_values, - "rrset_ttl": self.rrset_ttl, - } - - def as_api(self): - return {"rrset_values": self.rrset_values, "rrset_ttl": self.rrset_ttl} - - def from_dict(self, d): - self.rrset_name = d["rrset_name"] - self.rrset_type = d["rrset_type"] - self.rrset_values = d["rrset_values"] - self.rrset_ttl = d.get("rrset_ttl", 10800) - - def __repr__(self): - return "" % ( - self.rrset_name, - self.rrset_type, - self.rrset_values, - self.rrset_ttl, - ) - - def __eq__(self, other): - return ( - self.rrset_name == other.rrset_name - and self.rrset_type == other.rrset_type - and self.rrset_values == other.rrset_values - and self.rrset_ttl == other.rrset_ttl - ) - - def __hash__(self): - return hash(repr(self)) - - -class DomainsRecords(APIElement): - ENDPOINT = "livedns/domains/{fqdn}/records" - - def __init__(self, client, fqdn, records=None, fetch=True): - endpoint = self.ENDPOINT.format(fqdn=fqdn) - super().__init__(client, endpoint) - self.fqdn = fqdn - self.records = records - if self.records is None and fetch: - self.fetch() - else: - self.records = [] - - def save(self): - for r in self.records: - r.save() - - def from_dict(self, d): - # l is actually an array - if isinstance(d, dict): - l = d["records"] - else: - l = d - logging.getLogger("gandiAPI").debug(l) - self.records = [Record(self.client, self.fqdn, **r) for r in l] - - def as_dict(self): - return [r.as_dict() for r in self.records] - - def __repr__(self): - return "" % (self.fqdn, self.records) diff --git a/main.py b/main.py index 0dd349c..b5e4b1d 100644 --- a/main.py +++ b/main.py @@ -2,16 +2,17 @@ import logging import logging.config import pathlib import traceback +import socket import click import toml import requests +from jinja2 import Environment, FileSystemLoader from re2oapi import Re2oAPIClient -from gandi import GandiAPIClient, DomainsRecords, Record RUN_PATH = pathlib.Path(__file__).parent - +CLIENT_HOSTNAME = socket.gethostname().split('.', 1)[0] @click.command() @click.option( @@ -25,7 +26,7 @@ RUN_PATH = pathlib.Path(__file__).parent ) def main(config_dir, dry_run, keep): logging.config.fileConfig(config_dir / "logging.conf") - logger = logging.getLogger("dns") + logger = logging.getLogger("re2o-mails") logger.debug("Fetching configuration from %s.", config_dir) config = toml.load(config_dir / "config.toml") re2o_client = Re2oAPIClient( @@ -34,148 +35,48 @@ def main(config_dir, dry_run, keep): config["Re2o"]["password"], use_tls=config["Re2o"]["use_TLS"], ) - zones = re2o_client.list("dns/zones") - default_ttl = re2o_client.view("preferences/optionalmachine").get( - "default_dns_ttl", 10800 - ) + users_emails = re2o_client.list("localemail/users") + users = filter(lambda x: x["local_email_enabled"], re2o_client.list("users/user")) + local_email_domain = re2o_client.get("preferences/optionaluser")["local_email_domain"] + env = Environment(loader=FileSystemLoader(config_dir)) - default_API_key = config["Gandi"]["API_KEY"] + generated_folder = config_dir / "generated" + generated_folder.mkdir(exist_ok=True) - for zone in zones: - # Re2o has zones names begining with '.'. It is a bit difficult to translate - # that into toml - name = zone["name"][1:] - logger.debug(zone) - try: - configured_zone = config["Gandi"]["zone"][name] - except KeyError as e: - logger.error("Could not find zone named %s in configuration.", e) - continue + virtual_alias = generated_folder / "virtual_alias" + virtual_mailbox = generated_folder / "virtual_mailbox" - key = configured_zone.get("API_KEY", default_API_key) - gandi_client = GandiAPIClient(key) - - logger.info("Fetching last update for zone %s.", name) - last_update_file = config_dir / "last_update" / "last_update_{}.toml".format(name) - last_update_file.touch(mode=0o644, exist_ok=True) - last_update = DomainsRecords(gandi_client, name, fetch=False) - try: - last_update.from_dict(toml.load(last_update_file)) - except Exception as e: - logger.warning("Could not retrieve last update.") - logger.debug(e) - logger.info("Fetching current records for zone %s.", name) - current_records = DomainsRecords(gandi_client, name) - logger.info("Fetching re2o records for zone %s.", name) - new_records = set() - - if zone["originv4"]: - new_records.add( - Record( - gandi_client, - name, - rrset_name="@", - rrset_type=Record.Types.A, - rrset_values=[zone["originv4"]["ipv4"]], - rrset_ttl=zone["soa"].get("ttl", None) or default_ttl, - ) - ) - - if zone["originv6"]: - new_records.add( - Record( - gandi_client, - name, - rrset_name="@", - rrset_type=Record.Types.AAAA, - rrset_values=[zone["originv6"]], - rrset_ttl=zone["soa"].get("ttl", None) or default_ttl, - ) - ) - - for record in zone["a_records"]: - new_records.add( - Record( - gandi_client, - name, - rrset_name=record["hostname"], - rrset_type=Record.Types.A, - rrset_values=[record["ipv4"]], - rrset_ttl=record.get("ttl", None) or default_ttl, - ) - ) - - for record in zone["aaaa_records"]: - new_records.add( - Record( - gandi_client, - name, - rrset_name=record["hostname"], - rrset_type=Record.Types.AAAA, - rrset_values=[ipv6["ipv6"] for ipv6 in record["ipv6"]], - rrset_ttl=record.get("ttl", None) or default_ttl, - ) - ) - - for record in zone["cname_records"]: - new_records.add( - Record( - gandi_client, - name, - rrset_name=record["hostname"], - rrset_type=Record.Types.CNAME, - # The dot is to conform with Gandi API - rrset_values=[record["alias"] + "."], - rrset_ttl=record.get("ttl", None) or default_ttl, - ) - ) - - # Delete records added by this script that are not in new_records. - to_be_deleted = set(last_update.records) & ( - set(current_records.records) - set(new_records) - ) - # Add records that are in new_records except if they are already there. - to_be_added = set(new_records) - set(current_records.records) - logger.debug("Re2o records are %r", new_records) - logger.debug("I will add : %r", to_be_added) - logger.debug("I will delete : %r", to_be_deleted) - - if not dry_run: - saved = set() - for r in to_be_deleted: - logger.info("Deleting record %r for zone %s.", r, name) - try: - r.delete() - except requests.exceptions.HTTPError as e: - logger.error("Failed to delete %r for zone %s: %s", r, name, e) - saved.add(r) - - for r in to_be_added: - logger.info("Adding record %r for zone %s.", r, name) - try: - r.save() - saved.add(r) - except requests.exceptions.HTTPError as e: - logger.error("Failed to add %r for zone %s: %s", r, name, e) - logger.debug("Saving update for zone %s.", name) - - # the new last_update file should contain the old one, plus the ones registered on re2o, minus the ones that should have been removed and were not saved - managed = (set(last_update.records) | set(new_records)) - ( - to_be_deleted - saved - ) - with last_update_file.open("w") as f: - toml.dump({"records": [r.as_dict() for r in managed]}, f) + template_alias = env.get_template('templates/virtual_alias.j2') + template_mailbox = env.get_template('templates/virtual_mailbox.j2') + with virtual_mailbox.open(mode='w') as f_mailbox: + rendered_mailbox = template_mailbox.render(users=users, local_email_domain=local_email_domain) + if dry_run: + logging.info("New virtual mailboxes would be : %s", rendered_mailbox) else: - logger.info("This is a dry run for zone %s.", name) - logger.info("Records to be deleted : %r", to_be_deleted) - logger.info("Records to be added : %r", to_be_added) + logging.info("Writing virtual mailboxes") + f_mailbox.write(rendered_mailbox) + + with virtual_alias.open(mode='w') as f_alias: + rendered_alias = template_alias.render(users_emails=users_emails, local_email_domain=local_email_domain) + if dry_run: + logging.info("New virtual aliases would be : %s", rendered_alias) + else: + logging.info("Writing virtual aliases") + f_alias.write(rendered_alias) + + if not dry_run: + logging.info("Generating new maps") + call(["/usr/sbin/postmap", str(virtual_alias)], stdout=DEVNULL) + call(["/usr/sbin/postmap", str(virtual_mailbox)], stdout=DEVNULL) + logging.info("Reloading postfix") + call(["/usr/sbin/postfix", "reload"]) if not keep and not dry_run: for service in re2o_client.list("services/regen/"): if ( - service["hostname"] == client_hostname - and service["service_name"] == "dns" + service["hostname"] == CLIENT_HOSTNAME + and service["service_name"] == "mail-server" and service["need_regen"] ): re2o_client.patch(service["api_url"], data={"need_regen": False}) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3485359..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -certifi==2019.11.28 -chardet==3.0.4 -click==7.1.1 -idna==2.9 -iso8601==0.1.12 -requests==2.23.0 -toml==0.10.0 -urllib3==1.25.8 diff --git a/templates/virtual_alias.j2 b/templates/virtual_alias.j2 new file mode 100644 index 0000000..edf9201 --- /dev/null +++ b/templates/virtual_alias.j2 @@ -0,0 +1,11 @@ +# List of virtual aliases. Do not edit as it is generated by re2o. + +{% for user in users_emails %} +{% for alias in user.email_address %} +{% if user.local_email_redirect %} +{{alias.complete_email_address}} {{user.email|lower}} +{% else %} +{{alias.complete_email_address}} {{alias.user|lower}}{{domain}} +{% endif %} +{% endfor %} +{% endfor %} diff --git a/templates/virtual_mailbox.j2 b/templates/virtual_mailbox.j2 new file mode 100644 index 0000000..b811e7c --- /dev/null +++ b/templates/virtual_mailbox.j2 @@ -0,0 +1,5 @@ +# List of virtual mailboxes. Do not edit as it is generated by re2o. + +{% for user in users %} +{{user.pseudo}}{{local_email_domain}} whatever +{% endfor %}