New api and sync to mailman script

.gitignore

@@ -1,3 +1,4 @@
 config.ini
 **/__pycache__/**
 **.list
+generated

README.md

@@ -7,3 +7,67 @@ This service uses Re2o API to generate mailing member files.
 
 * python3
 * requirements in https://gitlab.federez.net/re2o/re2oapi
+
+## Configuration
+
+You need to copy the config.ini.example file into config.ini.
+
+### Re2o section
+
+The re2o section defines parameter to connect to re2o api.
+
+| Parameter   | Description                   | Default value      |
+|-------------|-------------------------------|--------------------|
+| `hostname`  | hostname of the re2o instance | `re2o.example.net` |
+| `username`  | username for re2o api         | `my_api_username`  |
+| `password`  | password for re2o api         | `my_api_password`  |
+
+### Mailman section
+
+| Parameter  | Description              | Default value    |
+|------------|--------------------------|------------------|
+| `url`      | url for mailman api      | `localhost:8001` |
+| `username` | username for mailman api | `restadmin`      |
+| `password` | password for mailman api | `restpassword`   |
+| `domain`   | domain for mailing lists | `example.net`    |
+
+### Sections for mailing-lists
+
+For each mailing-list you want to synchronise, you need to create a section. The section name should be one of the mailing retourned by re2o. Re2o returns :
+
+ * mails for all the adherents (`adherents`)
+ * mails for each group
+ * mails for each club
+
+For each section, you can have two parameters : 
+
+| Parameter   | Description                                                                               | Default value |
+|-------------|-------------------------------------------------------------------------------------------|---------------|
+| `activate`  | If yes, the mailing will be synchronised. `no` is equivalent to no section at all         | `no`          |
+| `list_name` | list name (without domain) on mailman. If not given, the section name is taken by default | section name  |
+
+### Example
+```
+[Re2o]
+hostname = re2o.rezometz.org
+username = service-daemon
+password = secret
+
+[Mailman]
+url = localhost:8001
+username = restadmin
+password = secret
+domain = rezometz.org
+
+[adherents]
+activate = yes
+
+[rezo]
+activate = yes
+
+[rezotage]
+activate = yes
+list_name = rezo-admin
+```
+
+3 mailings are generated : one which is adherents@rezometz.org with all adherents, one which is is rezo@rezometz.org with the group rezo and the last one is rezo-admin@rezometz.org with the group rezotage.

config.ini.example

@@ -3,8 +3,15 @@ hostname = re2o.example.net
 username = my_api_username
 password = my_api_password
 
+[Mailman] # if using sync_adherents_mailman.py
+url = localhost:8001
+username = restadmin
+password = restpassword
+domain = example.net
+
 [mailing-name1]
 activate = yes
 
 [mailing-name2]
 activate = no
+list_name = myml # if mailman name is different from re2o name

main.py

@@ -1,36 +1,44 @@
 from configparser import ConfigParser
 import socket
 import datetime
+import os
+import argparse
 
 from re2oapi import Re2oAPIClient
 
+path = os.path.dirname(os.path.abspath(__file__))
+
 config = ConfigParser()
-config.read('config.ini')
+config.read(path + '/config.ini')
 
 api_hostname = config.get('Re2o', 'hostname')
 api_password = config.get('Re2o', 'password')
 api_username = config.get('Re2o', 'username')
 
 
+
 fallback = config.getboolean('DEFAULT', 'activate', fallback=False)
 
 
 def write_generic_members_file(ml_name, members):
     if config.getboolean(ml_name, 'activate', fallback=fallback):
-        members = "\n".join(m['email'] for m in members)
+        members = "\n".join(m['get_mail'] for m in members)
-        filename = 'ml.{name}.list'.format(name=ml_name)
+        filename = path + '/generated/ml.{name}.list'.format(name=ml_name)
         with open(filename, 'w+') as f:
             f.write(members)
+        print("[OK] File for mailing list {} has been generated".format(ml_name))
+    else:
+        print("[INFO] Mailing list {} from re2o is not activated. Skipping.".format(ml_name))
 
 
 def write_standard_members_files(api_client):
-    for ml in api_client.list_mailingstandard():
+    for ml in api_client.list("mailing/standard"):
         write_generic_members_file(ml['name'], ml['members'])
 
 
 def write_club_members_files(api_client):
     fallback = config.get('DEFAULT', 'activate', fallback=False)
-    for ml in api_client.list_mailingclub():
+    for ml in api_client.list("mailing/club"):
         write_generic_members_file(ml['name'], ml['members'])
         write_generic_members_file(ml['name']+'-admin', ml['members'])
 
@@ -38,11 +46,31 @@ def write_club_members_files(api_client):
 api_client = Re2oAPIClient(api_hostname, api_username, api_password)
 client_hostname = socket.gethostname().split('.', 1)[0]
 
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--force", help="Force files regeneration", action="store_true")
+    args = parser.parse_args()
 
-for service in api_client.list_servicesregen():
+    if not os.path.exists(os.path.dirname(path + "/generated/")):
-#    if service['hostname'] == client_hostname and \
+        print("[WARN] generated directory does not exist")
-#            service['service_name'] == 'dns' and \
+        try:
-#            service['need_regen']:
+            os.makedirs(os.path.dirname(path + "/generated/"))
+        except Exception as e:
+            print("[ERROR] Impossible to create generated directory. Error was {}".format(e))
+
+    for service in api_client.list("services/regen/"):
+        if service['hostname'] == client_hostname and service['service_name'] == 'mailing':
+            if service['need_regen'] or args.force:
+                print("[..] Regenerating service {}".format(service['service_name']))
                 write_standard_members_files(api_client)
                 write_club_members_files(api_client)
-#        api_client.patch(service['api_url'], data={'need_regen': False})
+                api_client.patch(service['api_url'], data={'need_regen': False})
+
+                ## Write that the files have changed, for other scripts
+                filename = path + "/generated/changed"
+                with open(filename, "w+") as f:
+                    f.write("1")
+
+                print("[OK] Service {} has been regenerated.".format(service['service_name']))
+            else:
+                print("[OK] No service needed regeneration")

re2oapi

@@ -1 +1 @@
-Subproject commit 5b4523c797bffb90c998d5b424548756baa0c1d2
+Subproject commit ffaed921030deb6b6b01649709666807feb95370

sync_adherents_mailman.py

@@ -0,0 +1,54 @@
+from configparser import ConfigParser
+import requests
+import os
+import subprocess
+
+path = os.path.dirname(os.path.abspath(__file__))
+filename = path + "/generated/changed"
+
+config = ConfigParser()
+config.read(path + '/config.ini')
+
+mailman_url = config.get('Mailman', 'url')
+mailman_username = config.get('Mailman', 'username')
+mailman_password = config.get('Mailman', 'password')
+domain = config.get('Mailman', 'domain')
+
+changed = int(open(filename).read())
+
+if changed:
+    for section in config.sections():
+        if section not in ["Re2o", "Mailman"] and config.getboolean(section, 'activate'):
+            list_name = config.get(section, "list_name", fallback=section)
+            response1 = requests.get('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username, mailman_password))
+            if "entries" in response1.json():
+                entries = response1.json()['entries']
+                old_emails = [entry['email'] for entry in entries]
+                new_emails = open(path + "/generated/ml.{}.list".format(section)).read().split("\n")
+                emails_to_delete = [email for email in old_emails if email not in new_emails]
+                if emails_to_delete:
+                    print("[..] Deleting non members from list {}".format(list_name))
+                    response = requests.delete('http://{}/3.1/lists/{}@{}/roster/member'.format(mailman_url, list_name, domain), auth=(mailman_username,mailman_password), params={'emails': emails_to_delete})
+                    print("[OK] Non members where deleted from list {}".format(list_name))
+                else:
+                    print("[INFO] No member to delete for list {}".format(list_name))
+                emails_to_add = [email for email in new_emails if email not in old_emails]
+                if emails_to_add:
+                    print("[..] Adding members to list {}".format(list_name))
+                    with open(path + "/tmp", "w+") as f:
+                        for email in emails_to_add:
+                            f.write("{}\n".format(email))
+                    subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/tmp"])
+                    os.remove(path + "/tmp")
+                    print("[OK] Members added to list {}".format(list_name))
+                else:
+                    print("[INFO] No member to add to list {}".format(list_name))
+            else:
+                print("[..] Subscribing members to list {}".format(list_name))
+                subprocess.call(["mailman", "members", "{}@{}".format(list_name, domain), "-a", path + "/generated/ml.{}.list".format(section)])
+                print("[OK] List {} was regenerated".format(list_name))
+    
+    with open(filename, "w+") as f:
+        f.write("0")
+else:
+    print("Files have not changed since last execution. Skipping")