From 8d1f4f3fd894fd965f66f08ed8fc116d2bc84e49 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 16 Feb 2020 22:03:55 +0100 Subject: [PATCH] New api and sync to mailman script --- .gitignore | 1 + README.md | 64 +++++++++++++++++++++++++++++++++++++++ config.ini.example | 7 +++++ main.py | 52 +++++++++++++++++++++++-------- re2oapi | 2 +- sync_adherents_mailman.py | 54 +++++++++++++++++++++++++++++++++ 6 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 sync_adherents_mailman.py diff --git a/.gitignore b/.gitignore index 641c4cc..11f7192 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ config.ini **/__pycache__/** **.list +generated diff --git a/README.md b/README.md index e3c97f3..cc05206 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,67 @@ This service uses Re2o API to generate mailing member files. * python3 * requirements in https://gitlab.federez.net/re2o/re2oapi + +## Configuration + +You need to copy the config.ini.example file into config.ini. + +### Re2o section + +The re2o section defines parameter to connect to re2o api. + +| Parameter | Description | Default value | +|-------------|-------------------------------|--------------------| +| `hostname` | hostname of the re2o instance | `re2o.example.net` | +| `username` | username for re2o api | `my_api_username` | +| `password` | password for re2o api | `my_api_password` | + +### Mailman section + +| Parameter | Description | Default value | +|------------|--------------------------|------------------| +| `url` | url for mailman api | `localhost:8001` | +| `username` | username for mailman api | `restadmin` | +| `password` | password for mailman api | `restpassword` | +| `domain` | domain for mailing lists | `example.net` | + +### Sections for mailing-lists + +For each mailing-list you want to synchronise, you need to create a section. The section name should be one of the mailing retourned by re2o. Re2o returns : + + * mails for all the adherents (`adherents`) + * mails for each group + * mails for each club + +For each section, you can have two parameters : + +| Parameter | Description | Default value | +|-------------|-------------------------------------------------------------------------------------------|---------------| +| `activate` | If yes, the mailing will be synchronised. `no` is equivalent to no section at all | `no` | +| `list_name` | list name (without domain) on mailman. If not given, the section name is taken by default | section name | + +### Example +``` +[Re2o] +hostname = re2o.rezometz.org +username = service-daemon +password = secret + +[Mailman] +url = localhost:8001 +username = restadmin +password = secret +domain = rezometz.org + +[adherents] +activate = yes + +[rezo] +activate = yes + +[rezotage] +activate = yes +list_name = rezo-admin +``` + +3 mailings are generated : one which is adherents@rezometz.org with all adherents, one which is is rezo@rezometz.org with the group rezo and the last one is rezo-admin@rezometz.org with the group rezotage. diff --git a/config.ini.example b/config.ini.example index bd69271..cb1aead 100644 --- a/config.ini.example +++ b/config.ini.example @@ -3,8 +3,15 @@ hostname = re2o.example.net username = my_api_username password = my_api_password +[Mailman] # if using sync_adherents_mailman.py +url = localhost:8001 +username = restadmin +password = restpassword +domain = example.net + [mailing-name1] activate = yes [mailing-name2] activate = no +list_name = myml # if mailman name is different from re2o name diff --git a/main.py b/main.py index 4302e88..71baee5 100644 --- a/main.py +++ b/main.py @@ -1,36 +1,44 @@ from configparser import ConfigParser import socket import datetime +import os +import argparse from re2oapi import Re2oAPIClient +path = os.path.dirname(os.path.abspath(__file__)) + config = ConfigParser() -config.read('config.ini') +config.read(path + '/config.ini') api_hostname = config.get('Re2o', 'hostname') api_password = config.get('Re2o', 'password') api_username = config.get('Re2o', 'username') + fallback = config.getboolean('DEFAULT', 'activate', fallback=False) def write_generic_members_file(ml_name, members): if config.getboolean(ml_name, 'activate', fallback=fallback): - members = "\n".join(m['email'] for m in members) - filename = 'ml.{name}.list'.format(name=ml_name) + members = "\n".join(m['get_mail'] for m in members) + filename = path + '/generated/ml.{name}.list'.format(name=ml_name) with open(filename, 'w+') as f: f.write(members) + print("[OK] File for mailing list {} has been generated".format(ml_name)) + else: + print("[INFO] Mailing list {} from re2o is not activated. Skipping.".format(ml_name)) def write_standard_members_files(api_client): - for ml in api_client.list_mailingstandard(): + for ml in api_client.list("mailing/standard"): write_generic_members_file(ml['name'], ml['members']) def write_club_members_files(api_client): fallback = config.get('DEFAULT', 'activate', fallback=False) - for ml in api_client.list_mailingclub(): + for ml in api_client.list("mailing/club"): write_generic_members_file(ml['name'], ml['members']) write_generic_members_file(ml['name']+'-admin', ml['members']) @@ -38,11 +46,31 @@ def write_club_members_files(api_client): api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-f", "--force", help="Force files regeneration", action="store_true") + args = parser.parse_args() -for service in api_client.list_servicesregen(): -# if service['hostname'] == client_hostname and \ -# service['service_name'] == 'dns' and \ -# service['need_regen']: - write_standard_members_files(api_client) - write_club_members_files(api_client) -# api_client.patch(service['api_url'], data={'need_regen': False}) + if not os.path.exists(os.path.dirname(path + "/generated/")): + print("[WARN] generated directory does not exist") + try: + os.makedirs(os.path.dirname(path + "/generated/")) + except Exception as e: + print("[ERROR] Impossible to create generated directory. Error was {}".format(e)) + + for service in api_client.list("services/regen/"): + if service['hostname'] == client_hostname and service['service_name'] == 'mailing': + if service['need_regen'] or args.force: + print("[..] Regenerating service {}".format(service['service_name'])) + write_standard_members_files(api_client) + write_club_members_files(api_client) + api_client.patch(service['api_url'], data={'need_regen': False}) + + ## Write that the files have changed, for other scripts + filename = path + "/generated/changed" + with open(filename, "w+") as f: + f.write("1") + + print("[OK] Service {} has been regenerated.".format(service['service_name'])) + else: + print("[OK] No service needed regeneration") diff --git a/re2oapi b/re2oapi index 5b4523c..ffaed92 160000 --- a/re2oapi +++ b/re2oapi @@ -1 +1 @@ -Subproject commit 5b4523c797bffb90c998d5b424548756baa0c1d2 +Subproject commit ffaed921030deb6b6b01649709666807feb95370 diff --git a/sync_adherents_mailman.py b/sync_adherents_mailman.py new file mode 100644 index 0000000..b945a5d --- /dev/null +++ b/sync_adherents_mailman.py @@ -0,0 +1,54 @@ +from configparser import ConfigParser +import requests +import os +import subprocess + +path = os.path.dirname(os.path.abspath(__file__)) +filename = path + "/generated/changed" + +config = ConfigParser() +config.read(path + '/config.ini') + +mailman_url = config.get('Mailman', 'url') +mailman_username = config.get('Mailman', 'username') +mailman_password = config.get('Mailman', 'password') +domain = config.get('Mailman', 'domain') + +changed = int(open(filename).read()) + +if changed: + for section in config.sections(): + if section not in ["Re2o", "Mailman"] and config.getboolean(section, 'activate'): + list_name = config.get(section, "list_name", fallback=section) + response1 = requests.get('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username, mailman_password)) + if "entries" in response1.json(): + entries = response1.json()['entries'] + old_emails = [entry['email'] for entry in entries] + new_emails = open(path + "/generated/ml.{}.list".format(section)).read().split("\n") + emails_to_delete = [email for email in old_emails if email not in new_emails] + if emails_to_delete: + print("[..] Deleting non members from list {}".format(list_name)) + response = requests.delete('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete}) + print("[OK] Non members where deleted from list {}".format(list_name)) + else: + print("[INFO] No member to delete for list {}".format(list_name)) + emails_to_add = [email for email in new_emails if email not in old_emails] + if emails_to_add: + print("[..] Adding members to list {}".format(list_name)) + with open(path + "/tmp", "w+") as f: + for email in emails_to_add: + f.write("{}\n".format(email)) + subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/tmp"]) + os.remove(path + "/tmp") + print("[OK] Members added to list {}".format(list_name)) + else: + print("[INFO] No member to add to list {}".format(list_name)) + else: + print("[..] Subscribing members to list {}".format(list_name)) + subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/generated/ml.{}.list".format(section)]) + print("[OK] List {} was regenerated".format(list_name)) + + with open(filename, "w+") as f: + f.write("0") +else: + print("Files have not changed since last execution. Skipping")