From 8d1f4f3fd894fd965f66f08ed8fc116d2bc84e49 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 16 Feb 2020 22:03:55 +0100 Subject: [PATCH 1/3] New api and sync to mailman script --- .gitignore | 1 + README.md | 64 +++++++++++++++++++++++++++++++++++++++ config.ini.example | 7 +++++ main.py | 52 +++++++++++++++++++++++-------- re2oapi | 2 +- sync_adherents_mailman.py | 54 +++++++++++++++++++++++++++++++++ 6 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 sync_adherents_mailman.py diff --git a/.gitignore b/.gitignore index 641c4cc..11f7192 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ config.ini **/__pycache__/** **.list +generated diff --git a/README.md b/README.md index e3c97f3..cc05206 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,67 @@ This service uses Re2o API to generate mailing member files. * python3 * requirements in https://gitlab.federez.net/re2o/re2oapi + +## Configuration + +You need to copy the config.ini.example file into config.ini. + +### Re2o section + +The re2o section defines parameter to connect to re2o api. + +| Parameter | Description | Default value | +|-------------|-------------------------------|--------------------| +| `hostname` | hostname of the re2o instance | `re2o.example.net` | +| `username` | username for re2o api | `my_api_username` | +| `password` | password for re2o api | `my_api_password` | + +### Mailman section + +| Parameter | Description | Default value | +|------------|--------------------------|------------------| +| `url` | url for mailman api | `localhost:8001` | +| `username` | username for mailman api | `restadmin` | +| `password` | password for mailman api | `restpassword` | +| `domain` | domain for mailing lists | `example.net` | + +### Sections for mailing-lists + +For each mailing-list you want to synchronise, you need to create a section. The section name should be one of the mailing retourned by re2o. Re2o returns : + + * mails for all the adherents (`adherents`) + * mails for each group + * mails for each club + +For each section, you can have two parameters : + +| Parameter | Description | Default value | +|-------------|-------------------------------------------------------------------------------------------|---------------| +| `activate` | If yes, the mailing will be synchronised. `no` is equivalent to no section at all | `no` | +| `list_name` | list name (without domain) on mailman. If not given, the section name is taken by default | section name | + +### Example +``` +[Re2o] +hostname = re2o.rezometz.org +username = service-daemon +password = secret + +[Mailman] +url = localhost:8001 +username = restadmin +password = secret +domain = rezometz.org + +[adherents] +activate = yes + +[rezo] +activate = yes + +[rezotage] +activate = yes +list_name = rezo-admin +``` + +3 mailings are generated : one which is adherents@rezometz.org with all adherents, one which is is rezo@rezometz.org with the group rezo and the last one is rezo-admin@rezometz.org with the group rezotage. diff --git a/config.ini.example b/config.ini.example index bd69271..cb1aead 100644 --- a/config.ini.example +++ b/config.ini.example @@ -3,8 +3,15 @@ hostname = re2o.example.net username = my_api_username password = my_api_password +[Mailman] # if using sync_adherents_mailman.py +url = localhost:8001 +username = restadmin +password = restpassword +domain = example.net + [mailing-name1] activate = yes [mailing-name2] activate = no +list_name = myml # if mailman name is different from re2o name diff --git a/main.py b/main.py index 4302e88..71baee5 100644 --- a/main.py +++ b/main.py @@ -1,36 +1,44 @@ from configparser import ConfigParser import socket import datetime +import os +import argparse from re2oapi import Re2oAPIClient +path = os.path.dirname(os.path.abspath(__file__)) + config = ConfigParser() -config.read('config.ini') +config.read(path + '/config.ini') api_hostname = config.get('Re2o', 'hostname') api_password = config.get('Re2o', 'password') api_username = config.get('Re2o', 'username') + fallback = config.getboolean('DEFAULT', 'activate', fallback=False) def write_generic_members_file(ml_name, members): if config.getboolean(ml_name, 'activate', fallback=fallback): - members = "\n".join(m['email'] for m in members) - filename = 'ml.{name}.list'.format(name=ml_name) + members = "\n".join(m['get_mail'] for m in members) + filename = path + '/generated/ml.{name}.list'.format(name=ml_name) with open(filename, 'w+') as f: f.write(members) + print("[OK] File for mailing list {} has been generated".format(ml_name)) + else: + print("[INFO] Mailing list {} from re2o is not activated. Skipping.".format(ml_name)) def write_standard_members_files(api_client): - for ml in api_client.list_mailingstandard(): + for ml in api_client.list("mailing/standard"): write_generic_members_file(ml['name'], ml['members']) def write_club_members_files(api_client): fallback = config.get('DEFAULT', 'activate', fallback=False) - for ml in api_client.list_mailingclub(): + for ml in api_client.list("mailing/club"): write_generic_members_file(ml['name'], ml['members']) write_generic_members_file(ml['name']+'-admin', ml['members']) @@ -38,11 +46,31 @@ def write_club_members_files(api_client): api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-f", "--force", help="Force files regeneration", action="store_true") + args = parser.parse_args() -for service in api_client.list_servicesregen(): -# if service['hostname'] == client_hostname and \ -# service['service_name'] == 'dns' and \ -# service['need_regen']: - write_standard_members_files(api_client) - write_club_members_files(api_client) -# api_client.patch(service['api_url'], data={'need_regen': False}) + if not os.path.exists(os.path.dirname(path + "/generated/")): + print("[WARN] generated directory does not exist") + try: + os.makedirs(os.path.dirname(path + "/generated/")) + except Exception as e: + print("[ERROR] Impossible to create generated directory. Error was {}".format(e)) + + for service in api_client.list("services/regen/"): + if service['hostname'] == client_hostname and service['service_name'] == 'mailing': + if service['need_regen'] or args.force: + print("[..] Regenerating service {}".format(service['service_name'])) + write_standard_members_files(api_client) + write_club_members_files(api_client) + api_client.patch(service['api_url'], data={'need_regen': False}) + + ## Write that the files have changed, for other scripts + filename = path + "/generated/changed" + with open(filename, "w+") as f: + f.write("1") + + print("[OK] Service {} has been regenerated.".format(service['service_name'])) + else: + print("[OK] No service needed regeneration") diff --git a/re2oapi b/re2oapi index 5b4523c..ffaed92 160000 --- a/re2oapi +++ b/re2oapi @@ -1 +1 @@ -Subproject commit 5b4523c797bffb90c998d5b424548756baa0c1d2 +Subproject commit ffaed921030deb6b6b01649709666807feb95370 diff --git a/sync_adherents_mailman.py b/sync_adherents_mailman.py new file mode 100644 index 0000000..b945a5d --- /dev/null +++ b/sync_adherents_mailman.py @@ -0,0 +1,54 @@ +from configparser import ConfigParser +import requests +import os +import subprocess + +path = os.path.dirname(os.path.abspath(__file__)) +filename = path + "/generated/changed" + +config = ConfigParser() +config.read(path + '/config.ini') + +mailman_url = config.get('Mailman', 'url') +mailman_username = config.get('Mailman', 'username') +mailman_password = config.get('Mailman', 'password') +domain = config.get('Mailman', 'domain') + +changed = int(open(filename).read()) + +if changed: + for section in config.sections(): + if section not in ["Re2o", "Mailman"] and config.getboolean(section, 'activate'): + list_name = config.get(section, "list_name", fallback=section) + response1 = requests.get('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username, mailman_password)) + if "entries" in response1.json(): + entries = response1.json()['entries'] + old_emails = [entry['email'] for entry in entries] + new_emails = open(path + "/generated/ml.{}.list".format(section)).read().split("\n") + emails_to_delete = [email for email in old_emails if email not in new_emails] + if emails_to_delete: + print("[..] Deleting non members from list {}".format(list_name)) + response = requests.delete('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete}) + print("[OK] Non members where deleted from list {}".format(list_name)) + else: + print("[INFO] No member to delete for list {}".format(list_name)) + emails_to_add = [email for email in new_emails if email not in old_emails] + if emails_to_add: + print("[..] Adding members to list {}".format(list_name)) + with open(path + "/tmp", "w+") as f: + for email in emails_to_add: + f.write("{}\n".format(email)) + subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/tmp"]) + os.remove(path + "/tmp") + print("[OK] Members added to list {}".format(list_name)) + else: + print("[INFO] No member to add to list {}".format(list_name)) + else: + print("[..] Subscribing members to list {}".format(list_name)) + subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/generated/ml.{}.list".format(section)]) + print("[OK] List {} was regenerated".format(list_name)) + + with open(filename, "w+") as f: + f.write("0") +else: + print("Files have not changed since last execution. Skipping") From 5c00e9c3aeea8f5e508432dce4ced8880152e3ae Mon Sep 17 00:00:00 2001 From: root Date: Thu, 20 Feb 2020 11:13:29 +0100 Subject: [PATCH 2/3] Override roster url --- README.md | 13 +++++++------ sync_adherents_mailman.py | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index cc05206..45a9aa7 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,13 @@ The re2o section defines parameter to connect to re2o api. ### Mailman section -| Parameter | Description | Default value | -|------------|--------------------------|------------------| -| `url` | url for mailman api | `localhost:8001` | -| `username` | username for mailman api | `restadmin` | -| `password` | password for mailman api | `restpassword` | -| `domain` | domain for mailing lists | `example.net` | +| Parameter | Description | Default value | +|--------------|-------------------------------------------|----------------------------------------------------------------------------------------------------| +| `url` | url for mailman api | `localhost:8001` | +| `username` | username for mailman api | `restadmin` | +| `password` | password for mailman api | `restpassword` | +| `domain` | domain for mailing lists | `example.net` | +| `roster_url` | roster url for getting and deleting email | None (`http://{mailman_url}/3.1/lists/{list_name}@{domain}/roster/member` will be used by default) | ### Sections for mailing-lists diff --git a/sync_adherents_mailman.py b/sync_adherents_mailman.py index b945a5d..5af986b 100644 --- a/sync_adherents_mailman.py +++ b/sync_adherents_mailman.py @@ -13,14 +13,14 @@ mailman_url = config.get('Mailman', 'url') mailman_username = config.get('Mailman', 'username') mailman_password = config.get('Mailman', 'password') domain = config.get('Mailman', 'domain') - +roster_url = config.get('Mailman', 'roster_url', fallback='http://{mailman_url}/3.1/lists/{list_name}@{domain}/roster/member') changed = int(open(filename).read()) if changed: for section in config.sections(): if section not in ["Re2o", "Mailman"] and config.getboolean(section, 'activate'): list_name = config.get(section, "list_name", fallback=section) - response1 = requests.get('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username, mailman_password)) + response1 = requests.get(roster_url.format(mailman_url=mailman_url, list_namelist_name, domain=domain), auth=(mailman_username, mailman_password)) if "entries" in response1.json(): entries = response1.json()['entries'] old_emails = [entry['email'] for entry in entries] @@ -28,7 +28,7 @@ if changed: emails_to_delete = [email for email in old_emails if email not in new_emails] if emails_to_delete: print("[..] Deleting non members from list {}".format(list_name)) - response = requests.delete('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete}) + response = requests.delete(roster_url.format(mailman_url=mailman_url, list_name=list_name, domain=domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete}) print("[OK] Non members where deleted from list {}".format(list_name)) else: print("[INFO] No member to delete for list {}".format(list_name)) From 567293b5a13de6ab4b3d0884f4c73b29173fd465 Mon Sep 17 00:00:00 2001 From: nanoy Date: Thu, 20 Feb 2020 11:26:36 +0100 Subject: [PATCH 3/3] Add cron example to README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 45a9aa7..ae5eb6d 100644 --- a/README.md +++ b/README.md @@ -72,3 +72,14 @@ list_name = rezo-admin ``` 3 mailings are generated : one which is adherents@rezometz.org with all adherents, one which is is rezo@rezometz.org with the group rezo and the last one is rezo-admin@rezometz.org with the group rezotage. + +## Setup with a cron + +You can setup an automatic regeneration with, for instance, the following command : + +``` +* */2 * * * root python3 /usr/local/mailing/main.py; python3 /usr/local/mailing/sync_adherents_mailman.py +``` + +in `/etc/cron.d/mailing`. The two scripts are executed every two hours in this case (to limit the number of requests on mailman api even if the second script is executed only if the first regenerates files). +