.gitignore

@@ -1,3 +1,4 @@
 config.ini
 **/__pycache__/**
 **.list
+generated

README.md

@@ -7,3 +7,79 @@ This service uses Re2o API to generate mailing member files.
 
 * python3
 * requirements in https://gitlab.federez.net/re2o/re2oapi
+
+## Configuration
+
+You need to copy the config.ini.example file into config.ini.
+
+### Re2o section
+
+The re2o section defines parameter to connect to re2o api.
+
+| Parameter   | Description                   | Default value      |
+|-------------|-------------------------------|--------------------|
+| `hostname`  | hostname of the re2o instance | `re2o.example.net` |
+| `username`  | username for re2o api         | `my_api_username`  |
+| `password`  | password for re2o api         | `my_api_password`  |
+
+### Mailman section
+
+| Parameter    | Description                               | Default value                                                                                      |
+|--------------|-------------------------------------------|----------------------------------------------------------------------------------------------------|
+| `url`        | url for mailman api                       | `localhost:8001`                                                                                   |
+| `username`   | username for mailman api                  | `restadmin`                                                                                        |
+| `password`   | password for mailman api                  | `restpassword`                                                                                     |
+| `domain`     | domain for mailing lists                  | `example.net`                                                                                      |
+| `roster_url` | roster url for getting and deleting email | None (`http://{mailman_url}/3.1/lists/{list_name}@{domain}/roster/member` will be used by default) |
+
+### Sections for mailing-lists
+
+For each mailing-list you want to synchronise, you need to create a section. The section name should be one of the mailing retourned by re2o. Re2o returns :
+
+ * mails for all the adherents (`adherents`)
+ * mails for each group
+ * mails for each club
+
+For each section, you can have two parameters : 
+
+| Parameter   | Description                                                                               | Default value |
+|-------------|-------------------------------------------------------------------------------------------|---------------|
+| `activate`  | If yes, the mailing will be synchronised. `no` is equivalent to no section at all         | `no`          |
+| `list_name` | list name (without domain) on mailman. If not given, the section name is taken by default | section name  |
+
+### Example
+```
+[Re2o]
+hostname = re2o.rezometz.org
+username = service-daemon
+password = secret
+
+[Mailman]
+url = localhost:8001
+username = restadmin
+password = secret
+domain = rezometz.org
+
+[adherents]
+activate = yes
+
+[rezo]
+activate = yes
+
+[rezotage]
+activate = yes
+list_name = rezo-admin
+```
+
+3 mailings are generated : one which is adherents@rezometz.org with all adherents, one which is is rezo@rezometz.org with the group rezo and the last one is rezo-admin@rezometz.org with the group rezotage.
+
+## Setup with a cron
+
+You can setup an automatic regeneration with, for instance, the following command :
+
+```
+* */2 * * * root python3 /usr/local/mailing/main.py; python3 /usr/local/mailing/sync_adherents_mailman.py
+```
+
+in `/etc/cron.d/mailing`. The two scripts are executed every two hours in this case (to limit the number of requests on mailman api even if the second script is executed only if the first regenerates files).
+

config.ini.example

@@ -3,8 +3,15 @@ hostname = re2o.example.net
 username = my_api_username
 password = my_api_password
 
+[Mailman] # if using sync_adherents_mailman.py
+url = localhost:8001
+username = restadmin
+password = restpassword
+domain = example.net
+
 [mailing-name1]
 activate = yes
 
 [mailing-name2]
 activate = no
+list_name = myml # if mailman name is different from re2o name

main.py

@@ -1,36 +1,44 @@
 from configparser import ConfigParser
 import socket
 import datetime
+import os
+import argparse
 
 from re2oapi import Re2oAPIClient
 
+path = os.path.dirname(os.path.abspath(__file__))
+
 config = ConfigParser()
-config.read('config.ini')
+config.read(path + '/config.ini')
 
 api_hostname = config.get('Re2o', 'hostname')
 api_password = config.get('Re2o', 'password')
 api_username = config.get('Re2o', 'username')
 
 
+
 fallback = config.getboolean('DEFAULT', 'activate', fallback=False)
 
 
 def write_generic_members_file(ml_name, members):
     if config.getboolean(ml_name, 'activate', fallback=fallback):
-        members = "\n".join(m['email'] for m in members)
+        members = "\n".join(m['get_mail'] for m in members)
-        filename = 'ml.{name}.list'.format(name=ml_name)
+        filename = path + '/generated/ml.{name}.list'.format(name=ml_name)
         with open(filename, 'w+') as f:
             f.write(members)
+        print("[OK] File for mailing list {} has been generated".format(ml_name))
+    else:
+        print("[INFO] Mailing list {} from re2o is not activated. Skipping.".format(ml_name))
 
 
 def write_standard_members_files(api_client):
-    for ml in api_client.list_mailingstandard():
+    for ml in api_client.list("mailing/standard"):
         write_generic_members_file(ml['name'], ml['members'])
 
 
 def write_club_members_files(api_client):
     fallback = config.get('DEFAULT', 'activate', fallback=False)
-    for ml in api_client.list_mailingclub():
+    for ml in api_client.list("mailing/club"):
         write_generic_members_file(ml['name'], ml['members'])
         write_generic_members_file(ml['name']+'-admin', ml['members'])
 
@@ -38,11 +46,31 @@ def write_club_members_files(api_client):
 api_client = Re2oAPIClient(api_hostname, api_username, api_password)
 client_hostname = socket.gethostname().split('.', 1)[0]
 
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--force", help="Force files regeneration", action="store_true")
+    args = parser.parse_args()
 
-for service in api_client.list_servicesregen():
+    if not os.path.exists(os.path.dirname(path + "/generated/")):
-#    if service['hostname'] == client_hostname and \
+        print("[WARN] generated directory does not exist")
-#            service['service_name'] == 'dns' and \
+        try:
-#            service['need_regen']:
+            os.makedirs(os.path.dirname(path + "/generated/"))
-    write_standard_members_files(api_client)
+        except Exception as e:
-    write_club_members_files(api_client)
+            print("[ERROR] Impossible to create generated directory. Error was {}".format(e))
-#        api_client.patch(service['api_url'], data={'need_regen': False})
+
+    for service in api_client.list("services/regen/"):
+        if service['hostname'] == client_hostname and service['service_name'] == 'mailing':
+            if service['need_regen'] or args.force:
+                print("[..] Regenerating service {}".format(service['service_name']))
+                write_standard_members_files(api_client)
+                write_club_members_files(api_client)
+                api_client.patch(service['api_url'], data={'need_regen': False})
+
+                ## Write that the files have changed, for other scripts
+                filename = path + "/generated/changed"
+                with open(filename, "w+") as f:
+                    f.write("1")
+
+                print("[OK] Service {} has been regenerated.".format(service['service_name']))
+            else:
+                print("[OK] No service needed regeneration")

re2oapi

@@ -1 +1 @@
-Subproject commit 5b4523c797bffb90c998d5b424548756baa0c1d2
+Subproject commit ffaed921030deb6b6b01649709666807feb95370

sync_adherents_mailman.py

@@ -0,0 +1,54 @@
+from configparser import ConfigParser
+import requests
+import os
+import subprocess
+
+path = os.path.dirname(os.path.abspath(__file__))
+filename = path + "/generated/changed"
+
+config = ConfigParser()
+config.read(path + '/config.ini')
+
+mailman_url = config.get('Mailman', 'url')
+mailman_username = config.get('Mailman', 'username')
+mailman_password = config.get('Mailman', 'password')
+domain = config.get('Mailman', 'domain')
+roster_url = config.get('Mailman', 'roster_url', fallback='http://{mailman_url}/3.1/lists/{list_name}@{domain}/roster/member')
+changed = int(open(filename).read())
+
+if changed:
+    for section in config.sections():
+        if section not in ["Re2o", "Mailman"] and config.getboolean(section, 'activate'):
+            list_name = config.get(section, "list_name", fallback=section)
+            response1 = requests.get(roster_url.format(mailman_url=mailman_url, list_namelist_name, domain=domain), auth=(mailman_username, mailman_password))
+            if "entries" in response1.json():
+                entries = response1.json()['entries']
+                old_emails = [entry['email'] for entry in entries]
+                new_emails = open(path + "/generated/ml.{}.list".format(section)).read().split("\n")
+                emails_to_delete = [email for email in old_emails if email not in new_emails]
+                if emails_to_delete:
+                    print("[..] Deleting non members from list {}".format(list_name))
+                    response = requests.delete(roster_url.format(mailman_url=mailman_url, list_name=list_name, domain=domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete})
+                    print("[OK] Non members where deleted from list {}".format(list_name))
+                else:
+                    print("[INFO] No member to delete for list {}".format(list_name))
+                emails_to_add = [email for email in new_emails if email not in old_emails]
+                if emails_to_add:
+                    print("[..] Adding members to list {}".format(list_name))
+                    with open(path + "/tmp", "w+") as f:
+                        for email in emails_to_add:
+                            f.write("{}\n".format(email))
+                    subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/tmp"])
+                    os.remove(path + "/tmp")
+                    print("[OK] Members added to list {}".format(list_name))
+                else:
+                    print("[INFO] No member to add to list {}".format(list_name))
+            else:
+                print("[..] Subscribing members to list {}".format(list_name))
+                subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/generated/ml.{}.list".format(section)])
+                print("[OK] List {} was regenerated".format(list_name))
+    
+    with open(filename, "w+") as f:
+        f.write("0")
+else:
+    print("Files have not changed since last execution. Skipping")