From 8e70bc5ece2e6dc637cdecde559869f356424b6c Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 27 Jun 2018 17:07:04 +0200 Subject: [PATCH 01/45] New api handling, and typos --- main.py | 5 +++-- re2oapi | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 main.py diff --git a/main.py b/main.py old mode 100644 new mode 100755 index a267c5c..1f032ba --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from configparser import ConfigParser import socket import datetime @@ -124,7 +125,7 @@ def write_dns_files(api_client): cname_records = "\n".join( template_cname.format(hostname=x['hostname'], - alias=x['alias']+extension=x['extension']) + alias=x['alias'],extension=x['extension']) for x in zone['cname_records'] ) @@ -153,7 +154,7 @@ api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] -for service in api_client.list_servicesregen(): +for service in api_client.view("services/regen/"): # if service['hostname'] == client_hostname and \ # service['service_name'] == 'dns' and \ # service['need_regen']: diff --git a/re2oapi b/re2oapi index 5b4523c..b0885f7 160000 --- a/re2oapi +++ b/re2oapi @@ -1 +1 @@ -Subproject commit 5b4523c797bffb90c998d5b424548756baa0c1d2 +Subproject commit b0885f7b84245fa22ccb1b91c00d03675c1cfafc From d321aeb2eaa3958959867c92e17532ee36d57a24 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 27 Jun 2018 18:07:27 +0200 Subject: [PATCH 02/45] bugfix --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 1f032ba..dc6f753 100755 --- a/main.py +++ b/main.py @@ -58,7 +58,7 @@ template_reverse = ("$TTL 2D\n" "{ptr_records}\n") def write_dns_files(api_client): - for zone in api_client.list_dnszones(): + for zone in api_client.view("dns/zones")['results']: zone_name = zone['name'][1:] now = datetime.datetime.now(datetime.timezone.utc) @@ -154,10 +154,10 @@ api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] -for service in api_client.view("services/regen/"): +for service in api_client.view("services/regen/")['results']: # if service['hostname'] == client_hostname and \ # service['service_name'] == 'dns' and \ # service['need_regen']: - write_dns_zone_file(api_client) + write_dns_files(api_client) write_dns_reverse_file(api_client) # api_client.patch(service['api_url'], data={'need_regen': False}) From be31f657f64a3824ce40c4afd4d82eda7d29a888 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 27 Jun 2018 18:34:03 +0200 Subject: [PATCH 03/45] support mutiple ipv6, and no origin v4 --- main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index dc6f753..b7941ab 100755 --- a/main.py +++ b/main.py @@ -76,7 +76,10 @@ def write_dns_files(api_client): expire=zone['soa']['expire'], ttl=zone['soa']['ttl']) - originv4 = template_originv4.format(ipv4=zone['originv4']['ipv4']) + if zone['originv4'] is not None: + originv4 = template_originv4.format(ipv4=zone['originv4']['ipv4']) + else: + originv4 = "" if zone['originv6'] is not None: originv6 = template_originv6.format(ipv6=zone['originv6']) else: @@ -119,8 +122,10 @@ def write_dns_files(api_client): aaaa_records = "\n".join( template_aaaa.format(hostname=x['hostname'], - ipv6=x['ipv6']) - for x in zone['aaaa_records'] if x['ipv6'] is not None + ipv6=ip['ipv6']) + for x in zone['aaaa_records'] + for ip in x['ipv6'] + if x['ipv6'] is not None ) cname_records = "\n".join( From 7cba2399e3024aff61f751ecd0c3558bcac43a4c Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 27 Jun 2018 19:12:54 +0200 Subject: [PATCH 04/45] stupid charlie, stupid --- main.py | 4 ++-- re2oapi | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b7941ab..c764821 100755 --- a/main.py +++ b/main.py @@ -58,7 +58,7 @@ template_reverse = ("$TTL 2D\n" "{ptr_records}\n") def write_dns_files(api_client): - for zone in api_client.view("dns/zones")['results']: + for zone in api_client.list("dns/zones"): zone_name = zone['name'][1:] now = datetime.datetime.now(datetime.timezone.utc) @@ -159,7 +159,7 @@ api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] -for service in api_client.view("services/regen/")['results']: +for service in api_client.list("services/regen/"): # if service['hostname'] == client_hostname and \ # service['service_name'] == 'dns' and \ # service['need_regen']: diff --git a/re2oapi b/re2oapi index b0885f7..6565b92 160000 --- a/re2oapi +++ b/re2oapi @@ -1 +1 @@ -Subproject commit b0885f7b84245fa22ccb1b91c00d03675c1cfafc +Subproject commit 6565b92f3bfc13d02b95888ae021f5bd6f7ef317 From 190cbfda1649c21d1b22362652846ae70dcfde2e Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Mon, 16 Jul 2018 12:58:16 +0200 Subject: [PATCH 05/45] First ugly working version of dns reverse, to clean --- main.py | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index c764821..e4acf5d 100755 --- a/main.py +++ b/main.py @@ -2,6 +2,7 @@ from configparser import ConfigParser import socket import datetime +import netaddr from re2oapi import Re2oAPIClient @@ -55,6 +56,8 @@ template_reverse = ("$TTL 2D\n" "\n" "{ns_records}\n" "\n" + "{mx_records}\n" + "\n" "{ptr_records}\n") def write_dns_files(api_client): @@ -149,11 +152,135 @@ def write_dns_files(api_client): with open(filename, 'w+') as f: f.write(zone_file_content) +def network_to_arpanets(nets): + """En ipv4, on ne pouvait définir + que des plages reverse en /24, /16 ou /8. Cette fonction vise à retourner + une liste des plages en tenant compte de ce critère (donc de taille + 32/24/16/8) + + Ne touche à rien pour l'IPv6. + """ + subnets = [] + for net in nets: + if not isinstance(net, netaddr.IPNetwork): + net = netaddr.IPNetwork(net) + # on fragmente les subnets + # dans les tailles qui vont bien. + if net.prefixlen > 24: + subnets.extend(net.subnet(32)) + elif net.prefixlen > 16: + subnets.extend(net.subnet(24)) + elif net.prefixlen > 8: + subnets.extend(net.subnet(16)) + else: + subnets.extend(net.subnet(8)) + return subnets + + +def get_arpa(cidr): + """ + Renvoie la zone DNS inverse correspondant au réseau donné. + """ + net = netaddr.IPNetwork(cidr) + byte_size = 8 if net.version == 4 else 4 + addr_bytes = 4 if net.version == 4 else 32 + net_class = max(((net.prefixlen - 1) // byte_size) + 1, 1) + return ".".join( + netaddr.IPAddress(net.first).reverse_dns.split('.')[addr_bytes - net_class:] + ) + +def get_ip_reverse(ip, prefix_length): + ip = netaddr.IPAddress(ip) + return '.'.join(ip.reverse_dns.split('.')[:prefix_length]) + + def write_dns_reverse_file(api_client): - pass - + for zone in api_client.list("dns/reverse-zones"): + now = datetime.datetime.now(datetime.timezone.utc) + serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) + + extension = zone['extension'] + soa_mail_fields = zone['soa']['mail'].split('@') + soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), + soa_mail_fields[1]) + ns_records = "\n".join( + template_ns.format(target=x['target']) + for x in zone['ns_records'] + ) + + mx_records = "\n".join( + template_mx.format(priority=x['priority'], + target=x['target']) + for x in zone['mx_records'] + ) + + ### Start with ipv4 reverse + net = network_to_arpanets(zone['cidrs'])[0] + # Prend la première adresse ip de la plage, sauf si une est fournie + _address = netaddr.IPAddress(net.first) + # retourne le reverse splitté. (un reverse ressemble à 0.136.231.138.in-addr.arpa.) + rev_dns_a = _address.reverse_dns.split('.')[:-1] + + # En v4, le reverse étant de la forme 0.136.231.138.in-addr.arpa., soit + # on a un /8, soit un /16, soit un /24. + if net.prefixlen == 8: + zone_name,prefix_length = ('.'.join(rev_dns_a[3:]), 3) + elif net.prefixlen == 16: + zone_name,prefix_length = ('.'.join(rev_dns_a[2:]), 2) + elif net.prefixlen == 24: + zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) + + soa = template_soa.format(zone=zone_name, + mail=soa_mail, + serial=serial, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl']) + ptr_records = "\n".join( + template_ptr.format(hostname=host['hostname']+extension, + target=get_ip_reverse(host['ipv4'],prefix_length)) + for host in zone['ptr_records'] if host['ipv4'] + ) + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) + + filename = 'dns.{zone}zone'.format(zone=zone_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) + + ### Continue with the ipv6 reverse + # hack because we do not have the /64 info + zone6_name = get_arpa(zone['prefix_v6']+"/64") + soa = template_soa.format(zone=zone6_name, + mail=soa_mail, + serial=serial, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl']) + + net = netaddr.IPNetwork(zone['prefix_v6']+"/64") + prefix_length = int((128 - net.prefixlen)/4) + ptr_records = "\n".join( + template_ptr.format(hostname=host['hostname']+extension, + target=get_ip_reverse(ip['ipv6'],prefix_length)) + for host in zone['ptr_v6_records'] for ip in host['ipv6'] + ) + + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) + + + filename = 'dns.{zone}zone'.format(zone=zone6_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) api_client = Re2oAPIClient(api_hostname, api_username, api_password) @@ -163,6 +290,6 @@ for service in api_client.list("services/regen/"): # if service['hostname'] == client_hostname and \ # service['service_name'] == 'dns' and \ # service['need_regen']: - write_dns_files(api_client) - write_dns_reverse_file(api_client) + #write_dns_files(api_client) + write_dns_reverse_file(api_client) # api_client.patch(service['api_url'], data={'need_regen': False}) From ccecfa4e193363fc9b20819e60ee2dcb5d371c50 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Mon, 16 Jul 2018 20:54:31 +0200 Subject: [PATCH 06/45] Cleaning up --- main.py | 172 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 92 insertions(+), 80 deletions(-) diff --git a/main.py b/main.py index e4acf5d..90a4fe7 100755 --- a/main.py +++ b/main.py @@ -152,52 +152,28 @@ def write_dns_files(api_client): with open(filename, 'w+') as f: f.write(zone_file_content) -def network_to_arpanets(nets): - """En ipv4, on ne pouvait définir - que des plages reverse en /24, /16 ou /8. Cette fonction vise à retourner - une liste des plages en tenant compte de ce critère (donc de taille - 32/24/16/8) - - Ne touche à rien pour l'IPv6. - """ - subnets = [] - for net in nets: - if not isinstance(net, netaddr.IPNetwork): - net = netaddr.IPNetwork(net) - # on fragmente les subnets - # dans les tailles qui vont bien. - if net.prefixlen > 24: - subnets.extend(net.subnet(32)) - elif net.prefixlen > 16: - subnets.extend(net.subnet(24)) - elif net.prefixlen > 8: - subnets.extend(net.subnet(16)) - else: - subnets.extend(net.subnet(8)) - return subnets - - -def get_arpa(cidr): - """ - Renvoie la zone DNS inverse correspondant au réseau donné. - """ - net = netaddr.IPNetwork(cidr) - byte_size = 8 if net.version == 4 else 4 - addr_bytes = 4 if net.version == 4 else 32 - net_class = max(((net.prefixlen - 1) // byte_size) + 1, 1) - return ".".join( - netaddr.IPAddress(net.first).reverse_dns.split('.')[addr_bytes - net_class:] - ) def get_ip_reverse(ip, prefix_length): + """ Truncate an ip address given a prefix length """ ip = netaddr.IPAddress(ip) return '.'.join(ip.reverse_dns.split('.')[:prefix_length]) def write_dns_reverse_file(api_client): - for zone in api_client.list("dns/reverse-zones"): + """ Generate the reverve file for each reverse zone (= IpType) + For each IpType, we generate both an Ipv4 reverse and a v6. + The main issue is that we have to aggregat some IpTypes together, + because a reverse file can only describe a /24,/16 or /8. + """ + # We need to rember which zone file we already created for the v6 + # because some iptype may share the same prefix + # in which case we must append to the file zone already created + zone_v6 = [] + + for zone in api_client.list("dns/reverse-zones"): + # We start by defining the soa, ns, mx which are comon to v4/v6 now = datetime.datetime.now(datetime.timezone.utc) serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) @@ -215,47 +191,70 @@ def write_dns_reverse_file(api_client): target=x['target']) for x in zone['mx_records'] ) - - ### Start with ipv4 reverse - net = network_to_arpanets(zone['cidrs'])[0] - # Prend la première adresse ip de la plage, sauf si une est fournie - _address = netaddr.IPAddress(net.first) - # retourne le reverse splitté. (un reverse ressemble à 0.136.231.138.in-addr.arpa.) - rev_dns_a = _address.reverse_dns.split('.')[:-1] - # En v4, le reverse étant de la forme 0.136.231.138.in-addr.arpa., soit - # on a un /8, soit un /16, soit un /24. - if net.prefixlen == 8: - zone_name,prefix_length = ('.'.join(rev_dns_a[3:]), 3) - elif net.prefixlen == 16: - zone_name,prefix_length = ('.'.join(rev_dns_a[2:]), 2) - elif net.prefixlen == 24: - zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) - soa = template_soa.format(zone=zone_name, - mail=soa_mail, - serial=serial, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl']) - ptr_records = "\n".join( - template_ptr.format(hostname=host['hostname']+extension, - target=get_ip_reverse(host['ipv4'],prefix_length)) - for host in zone['ptr_records'] if host['ipv4'] - ) - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) + ### We start with the v4 + # We setup the network from the cidrs of the IpType + + # For the ipv4, we need to agregate the subnets together, because + # we can only have reverse for /24, /16 and /8. + subnets = [] + for net in zone['cidrs']: + net = netaddr.IPNetwork(net) + # on fragmente les subnets + # dans les tailles qui vont bien. + if net.prefixlen > 24: + subnets.extend(net.subnet(32)) + elif net.prefixlen > 16: + subnets.extend(net.subnet(24)) + elif net.prefixlen > 8: + subnets.extend(net.subnet(16)) + else: + subnets.extend(net.subnet(8)) + + for subnet in subnets: + # Then, using the first ip address of the subnet and the + # prefix length, we can obtain the name of the reverse zone + _address = netaddr.IPAddress(subnet.first) + rev_dns_a = _address.reverse_dns.split('.')[:-1] + if subnet.prefixlen == 8: + zone_name,prefix_length = ('.'.join(rev_dns_a[3:]), 3) + elif subnet.prefixlen == 16: + zone_name,prefix_length = ('.'.join(rev_dns_a[2:]), 2) + elif subnet.prefixlen == 24: + zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) + + soa = template_soa.format(zone=zone_name, + mail=soa_mail, + serial=serial, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl']) + ptr_records = "\n".join( + template_ptr.format(hostname=host['hostname']+extension, + target=get_ip_reverse(host['ipv4'],prefix_length)) + for host in zone['ptr_records'] if host['ipv4'] in subnet + ) + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) + + filename = 'dns.{zone}zone'.format(zone=zone_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) - filename = 'dns.{zone}zone'.format(zone=zone_name) - with open(filename, 'w+') as f: - f.write(zone_file_content) ### Continue with the ipv6 reverse # hack because we do not have the /64 info - zone6_name = get_arpa(zone['prefix_v6']+"/64") + net = netaddr.IPNetwork(zone['prefix_v6']+"/64") + net_class = max(((net.prefixlen - 1) // 4) + 1, 1) + zone6_name = ".".join( + netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] + ) + + soa = template_soa.format(zone=zone6_name, mail=soa_mail, serial=serial, @@ -271,16 +270,29 @@ def write_dns_reverse_file(api_client): target=get_ip_reverse(ip['ipv6'],prefix_length)) for host in zone['ptr_v6_records'] for ip in host['ipv6'] ) - - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) + if zone6_name in zone_v6: + # we already created the file, we ignore the soa + zone_file_content = template_reverse.format(soa="", + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) - filename = 'dns.{zone}zone'.format(zone=zone6_name) - with open(filename, 'w+') as f: - f.write(zone_file_content) + filename = 'dns.{zone}zone'.format(zone=zone6_name) + with open(filename, 'a') as f: + f.write(zone_file_content) + else: + # we create the file from scratch + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) + + + filename = 'dns.{zone}zone'.format(zone=zone6_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) + zone_v6.append(zone6_name) api_client = Re2oAPIClient(api_hostname, api_username, api_password) From 2224e3b5639d2313e3bc1dc7968e8cbcbe281c43 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Mon, 16 Jul 2018 20:57:03 +0200 Subject: [PATCH 07/45] prefix v6 length from server --- main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 90a4fe7..80102ff 100755 --- a/main.py +++ b/main.py @@ -247,8 +247,8 @@ def write_dns_reverse_file(api_client): ### Continue with the ipv6 reverse - # hack because we do not have the /64 info - net = netaddr.IPNetwork(zone['prefix_v6']+"/64") + net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) + print(net) net_class = max(((net.prefixlen - 1) // 4) + 1, 1) zone6_name = ".".join( netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] @@ -263,7 +263,6 @@ def write_dns_reverse_file(api_client): expire=zone['soa']['expire'], ttl=zone['soa']['ttl']) - net = netaddr.IPNetwork(zone['prefix_v6']+"/64") prefix_length = int((128 - net.prefixlen)/4) ptr_records = "\n".join( template_ptr.format(hostname=host['hostname']+extension, From 4b293167e906098d04273b36d8426b2f97197e6d Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Mon, 16 Jul 2018 21:08:27 +0200 Subject: [PATCH 08/45] bug fixes --- main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index 80102ff..1b4e749 100755 --- a/main.py +++ b/main.py @@ -241,14 +241,13 @@ def write_dns_reverse_file(api_client): mx_records=mx_records, ptr_records = ptr_records) - filename = 'dns.{zone}zone'.format(zone=zone_name) + filename = 'dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) ### Continue with the ipv6 reverse net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) - print(net) net_class = max(((net.prefixlen - 1) // 4) + 1, 1) zone6_name = ".".join( netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] From a0b93b94e05f13a71c224706d956e86775623d48 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Tue, 7 Aug 2018 08:08:47 +0200 Subject: [PATCH 09/45] modules on crans --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 361b0a1..94389b4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "re2oapi"] path = re2oapi - url = https://gitlab.federez.net/re2o/re2oapi.git + url = https://gitlab.crans.org/nounous/re2o-re2oapi.git From fee0f35f45e74ef132dcefeea7184a16e224e4a3 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Tue, 7 Aug 2018 10:04:39 +0200 Subject: [PATCH 10/45] no generation if no ptr record --- main.py | 166 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 82 deletions(-) diff --git a/main.py b/main.py index 1b4e749..7739903 100755 --- a/main.py +++ b/main.py @@ -148,7 +148,7 @@ def write_dns_files(api_client): aaaa_records=aaaa_records, cname_records=cname_records) - filename = 'dns.{zone}.zone'.format(zone=zone_name) + filename = 'generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) @@ -198,108 +198,110 @@ def write_dns_reverse_file(api_client): # For the ipv4, we need to agregate the subnets together, because # we can only have reverse for /24, /16 and /8. - subnets = [] - for net in zone['cidrs']: - net = netaddr.IPNetwork(net) - # on fragmente les subnets - # dans les tailles qui vont bien. - if net.prefixlen > 24: - subnets.extend(net.subnet(32)) - elif net.prefixlen > 16: - subnets.extend(net.subnet(24)) - elif net.prefixlen > 8: - subnets.extend(net.subnet(16)) - else: - subnets.extend(net.subnet(8)) + if zone['ptr_records']: + subnets = [] + for net in zone['cidrs']: + net = netaddr.IPNetwork(net) + # on fragmente les subnets + # dans les tailles qui vont bien. + if net.prefixlen > 24: + subnets.extend(net.subnet(32)) + elif net.prefixlen > 16: + subnets.extend(net.subnet(24)) + elif net.prefixlen > 8: + subnets.extend(net.subnet(16)) + else: + subnets.extend(net.subnet(8)) - for subnet in subnets: - # Then, using the first ip address of the subnet and the - # prefix length, we can obtain the name of the reverse zone - _address = netaddr.IPAddress(subnet.first) - rev_dns_a = _address.reverse_dns.split('.')[:-1] - if subnet.prefixlen == 8: - zone_name,prefix_length = ('.'.join(rev_dns_a[3:]), 3) - elif subnet.prefixlen == 16: - zone_name,prefix_length = ('.'.join(rev_dns_a[2:]), 2) - elif subnet.prefixlen == 24: - zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) + for subnet in subnets: + # Then, using the first ip address of the subnet and the + # prefix length, we can obtain the name of the reverse zone + _address = netaddr.IPAddress(subnet.first) + rev_dns_a = _address.reverse_dns.split('.')[:-1] + if subnet.prefixlen == 8: + zone_name,prefix_length = ('.'.join(rev_dns_a[3:]), 3) + elif subnet.prefixlen == 16: + zone_name,prefix_length = ('.'.join(rev_dns_a[2:]), 2) + elif subnet.prefixlen == 24: + zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) - soa = template_soa.format(zone=zone_name, + soa = template_soa.format(zone=zone_name, + mail=soa_mail, + serial=serial, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl']) + ptr_records = "\n".join( + template_ptr.format(hostname=host['hostname']+extension, + target=get_ip_reverse(host['ipv4'],prefix_length)) + for host in zone['ptr_records'] if host['ipv4'] in subnet + ) + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) + + filename = 'dns.{zone}.zone'.format(zone=zone_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) + + + ### Continue with the ipv6 reverse + if zone['ptr_v6_records']: + net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) + net_class = max(((net.prefixlen - 1) // 4) + 1, 1) + zone6_name = ".".join( + netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] + ) + + + soa = template_soa.format(zone=zone6_name, mail=soa_mail, serial=serial, refresh=zone['soa']['refresh'], retry=zone['soa']['retry'], expire=zone['soa']['expire'], ttl=zone['soa']['ttl']) + + prefix_length = int((128 - net.prefixlen)/4) ptr_records = "\n".join( template_ptr.format(hostname=host['hostname']+extension, - target=get_ip_reverse(host['ipv4'],prefix_length)) - for host in zone['ptr_records'] if host['ipv4'] in subnet + target=get_ip_reverse(ip['ipv6'],prefix_length)) + for host in zone['ptr_v6_records'] for ip in host['ipv6'] ) - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) - - filename = 'dns.{zone}.zone'.format(zone=zone_name) - with open(filename, 'w+') as f: - f.write(zone_file_content) + if zone6_name in zone_v6: + # we already created the file, we ignore the soa + zone_file_content = template_reverse.format(soa="", + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) - ### Continue with the ipv6 reverse - net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) - net_class = max(((net.prefixlen - 1) // 4) + 1, 1) - zone6_name = ".".join( - netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] - ) - - - soa = template_soa.format(zone=zone6_name, - mail=soa_mail, - serial=serial, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl']) - - prefix_length = int((128 - net.prefixlen)/4) - ptr_records = "\n".join( - template_ptr.format(hostname=host['hostname']+extension, - target=get_ip_reverse(ip['ipv6'],prefix_length)) - for host in zone['ptr_v6_records'] for ip in host['ipv6'] - ) - if zone6_name in zone_v6: - # we already created the file, we ignore the soa - zone_file_content = template_reverse.format(soa="", - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) + filename = 'dns.{zone}zone'.format(zone=zone6_name) + with open(filename, 'a') as f: + f.write(zone_file_content) + else: + # we create the file from scratch + zone_file_content = template_reverse.format(soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records) - filename = 'dns.{zone}zone'.format(zone=zone6_name) - with open(filename, 'a') as f: - f.write(zone_file_content) - else: - # we create the file from scratch - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) - - - filename = 'dns.{zone}zone'.format(zone=zone6_name) - with open(filename, 'w+') as f: - f.write(zone_file_content) - zone_v6.append(zone6_name) + filename = 'dns.{zone}zone'.format(zone=zone6_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) + zone_v6.append(zone6_name) api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] -for service in api_client.list("services/regen/"): +#for service in api_client.list("services/regen/"): # if service['hostname'] == client_hostname and \ # service['service_name'] == 'dns' and \ # service['need_regen']: - #write_dns_files(api_client) - write_dns_reverse_file(api_client) +write_dns_files(api_client) +write_dns_reverse_file(api_client) # api_client.patch(service['api_url'], data={'need_regen': False}) From 84cf40c2127fe6ec6079823c77d7128406d485b3 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Tue, 7 Aug 2018 10:06:31 +0200 Subject: [PATCH 11/45] moving gen files in subfolder --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 7739903..012acfb 100755 --- a/main.py +++ b/main.py @@ -242,7 +242,7 @@ def write_dns_reverse_file(api_client): mx_records=mx_records, ptr_records = ptr_records) - filename = 'dns.{zone}.zone'.format(zone=zone_name) + filename = 'generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) @@ -278,7 +278,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'dns.{zone}zone'.format(zone=zone6_name) + filename = 'generated/dns.{zone}zone'.format(zone=zone6_name) with open(filename, 'a') as f: f.write(zone_file_content) else: @@ -289,7 +289,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'dns.{zone}zone'.format(zone=zone6_name) + filename = 'generated/dns.{zone}zone'.format(zone=zone6_name) with open(filename, 'w+') as f: f.write(zone_file_content) zone_v6.append(zone6_name) From 7b37ae54202fe6732ba369c104952a84f4aeaf52 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Tue, 7 Aug 2018 10:52:56 +0200 Subject: [PATCH 12/45] fix templates --- main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 012acfb..8824b9d 100755 --- a/main.py +++ b/main.py @@ -13,7 +13,8 @@ api_hostname = config.get('Re2o', 'hostname') api_password = config.get('Re2o', 'password') api_username = config.get('Re2o', 'username') -template_soa = ("{zone} IN SOA ns.{zone}. {mail} (\n" +template_soa = ("$ORIGIN {zone}.\n" + "@ IN SOA {ns} {mail} (\n" " {serial} ; serial\n" " {refresh} ; refresh\n" " {retry} ; retry\n" @@ -70,10 +71,14 @@ def write_dns_files(api_client): soa_mail_fields = zone['soa']['mail'].split('@') soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), soa_mail_fields[1]) - + if zone['ns_records']: + ns = zone['ns_records'][0]['target'] + else: + ns = "ns."+zone_name+"." soa = template_soa.format(zone=zone_name, mail=soa_mail, serial=serial, + ns=ns, refresh=zone['soa']['refresh'], retry=zone['soa']['retry'], expire=zone['soa']['expire'], @@ -176,8 +181,11 @@ def write_dns_reverse_file(api_client): # We start by defining the soa, ns, mx which are comon to v4/v6 now = datetime.datetime.now(datetime.timezone.utc) serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) - extension = zone['extension'] + if zone['ns_records']: + ns = zone['ns_records'][0]['target'] + else: + ns = "ns"+extension+"." soa_mail_fields = zone['soa']['mail'].split('@') soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), soa_mail_fields[1]) @@ -228,6 +236,7 @@ def write_dns_reverse_file(api_client): soa = template_soa.format(zone=zone_name, mail=soa_mail, serial=serial, + ns=ns, refresh=zone['soa']['refresh'], retry=zone['soa']['retry'], expire=zone['soa']['expire'], @@ -259,6 +268,7 @@ def write_dns_reverse_file(api_client): soa = template_soa.format(zone=zone6_name, mail=soa_mail, serial=serial, + ns=ns, refresh=zone['soa']['refresh'], retry=zone['soa']['retry'], expire=zone['soa']['expire'], From e00771992633d084c57e312f209887db2a7ff1a6 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 8 Aug 2018 11:17:12 +0200 Subject: [PATCH 13/45] template fixes --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 8824b9d..4831515 100755 --- a/main.py +++ b/main.py @@ -23,14 +23,14 @@ template_soa = ("$ORIGIN {zone}.\n" ")") template_originv4 = "@ IN A {ipv4}" template_originv6 = "@ IN AAAA {ipv6}" -template_ns = "@ IN NS {target}" -template_mx = "@ IN MX {priority} {target}" +template_ns = "@ IN NS {target}." +template_mx = "@ IN MX {priority} {target}." template_txt = "{field1} IN TXT {field2}" template_srv = "_{service}._{protocol}.{zone} {ttl} IN SRV {priority} {weight} {port} {target}" template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" template_cname = "{hostname} IN CNAME {alias}" -template_ptr = "{target} IN PTR {hostname}" +template_ptr = "{target} IN PTR {hostname}." template_zone = ("$TTL 2D\n" "{soa}\n" From a56e9c444cc743586ebdf7447103cf54f61fdaf8 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 8 Aug 2018 11:59:08 +0200 Subject: [PATCH 14/45] fix aliases --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 4831515..08fb84b 100755 --- a/main.py +++ b/main.py @@ -29,7 +29,7 @@ template_txt = "{field1} IN TXT {field2}" template_srv = "_{service}._{protocol}.{zone} {ttl} IN SRV {priority} {weight} {port} {target}" template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" -template_cname = "{hostname} IN CNAME {alias}" +template_cname = "{hostname} IN CNAME {alias}." template_ptr = "{target} IN PTR {hostname}." template_zone = ("$TTL 2D\n" @@ -138,7 +138,7 @@ def write_dns_files(api_client): cname_records = "\n".join( template_cname.format(hostname=x['hostname'], - alias=x['alias'],extension=x['extension']) + alias=x['alias']) for x in zone['cname_records'] ) From 3fc12f60070cd1e4dc4e0961cdc21dccb5272081 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 8 Aug 2018 11:59:24 +0200 Subject: [PATCH 15/45] ipv6 reverse fix --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 08fb84b..363b6af 100755 --- a/main.py +++ b/main.py @@ -262,7 +262,7 @@ def write_dns_reverse_file(api_client): net_class = max(((net.prefixlen - 1) // 4) + 1, 1) zone6_name = ".".join( netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] - ) + )[:-1] soa = template_soa.format(zone=zone6_name, @@ -288,7 +288,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'generated/dns.{zone}zone'.format(zone=zone6_name) + filename = 'generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'a') as f: f.write(zone_file_content) else: @@ -299,7 +299,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'generated/dns.{zone}zone'.format(zone=zone6_name) + filename = 'generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'w+') as f: f.write(zone_file_content) zone_v6.append(zone6_name) From 79b2bc36555b7f6748bc390fdb417662e31b0cba Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 8 Aug 2018 12:47:09 +0200 Subject: [PATCH 16/45] soa fix --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 363b6af..a0add79 100755 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ api_password = config.get('Re2o', 'password') api_username = config.get('Re2o', 'username') template_soa = ("$ORIGIN {zone}.\n" - "@ IN SOA {ns} {mail} (\n" + "@ IN SOA {ns}. {mail} (\n" " {serial} ; serial\n" " {refresh} ; refresh\n" " {retry} ; retry\n" From 4fcd0274bbec17a76ba820bb538ff5cd84d7cd6d Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Wed, 8 Aug 2018 19:00:33 +0200 Subject: [PATCH 17/45] enable refresh --- main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 8824b9d..0b70ef5 100755 --- a/main.py +++ b/main.py @@ -308,10 +308,10 @@ api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] -#for service in api_client.list("services/regen/"): -# if service['hostname'] == client_hostname and \ -# service['service_name'] == 'dns' and \ -# service['need_regen']: -write_dns_files(api_client) -write_dns_reverse_file(api_client) -# api_client.patch(service['api_url'], data={'need_regen': False}) +for service in api_client.list("services/regen/"): + if service['hostname'] == client_hostname and \ + service['service_name'] == 'dns' and \ + service['need_regen']: + write_dns_files(api_client) + write_dns_reverse_file(api_client) + api_client.patch(service['api_url'], data={'need_regen': False}) From 65e35fb65517d9410a63e1407abb9d4c37803f7a Mon Sep 17 00:00:00 2001 From: grisel-davy Date: Wed, 8 Aug 2018 20:07:15 +0200 Subject: [PATCH 18/45] ajout de l'option --force --- main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/main.py b/main.py index 2905bd8..a4e10fc 100755 --- a/main.py +++ b/main.py @@ -6,6 +6,8 @@ import netaddr from re2oapi import Re2oAPIClient +import sys + config = ConfigParser() config.read('config.ini') @@ -61,6 +63,11 @@ template_reverse = ("$TTL 2D\n" "\n" "{ptr_records}\n") + +for arg in sys.argv: + if arg=="--force": + write_dns_files(api_client) + def write_dns_files(api_client): for zone in api_client.list("dns/zones"): zone_name = zone['name'][1:] From a950989980b398856a9b4e0182b94fd3770c11bf Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Thu, 9 Aug 2018 01:26:46 +0200 Subject: [PATCH 19/45] sshfingerprint in DNS --- main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/main.py b/main.py index a4e10fc..e1a4829 100755 --- a/main.py +++ b/main.py @@ -33,6 +33,7 @@ template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" template_cname = "{hostname} IN CNAME {alias}." template_ptr = "{target} IN PTR {hostname}." +template_sshfp = "{hostname} SSHFP {algo} {type} {fp}" template_zone = ("$TTL 2D\n" "{soa}\n" @@ -42,6 +43,9 @@ template_zone = ("$TTL 2D\n" "\n" "{ns_records}\n" "\n" + "{fp_records}\n" + "\n" + "{ns_records}\n" "{mx_records}\n" "\n" "{txt_records}\n" @@ -104,7 +108,16 @@ def write_dns_files(api_client): template_ns.format(target=x['target']) for x in zone['ns_records'] ) + + fp_records = "\n".join( + template_sshfp.format(hostname=host['hostname'], algo=fp['algo_id'], type="1", fp=fp['hash']['1'] ) + + "\n" + + template_sshfp.format(hostname=host['hostname'], algo=fp['algo_id'], type="2", fp=fp['hash']['2'] ) + for host in zone['sshfp_records'] + for fp in host['sshfp'] + ) + mx_records = "\n".join( template_mx.format(priority=x['priority'], target=x['target']) @@ -153,6 +166,7 @@ def write_dns_files(api_client): originv4=originv4, originv6=originv6, ns_records=ns_records, + fp_records=fp_records, mx_records=mx_records, txt_records=txt_records, srv_records=srv_records, From b4e10b36f098bf14ee527c3e314df3d8cb2321c7 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Thu, 9 Aug 2018 14:47:49 +0200 Subject: [PATCH 20/45] abs path for gen files --- main.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index e1a4829..a80d893 100755 --- a/main.py +++ b/main.py @@ -7,6 +7,11 @@ import netaddr from re2oapi import Re2oAPIClient import sys +import os +import os + + +path =(os.path.dirname(os.path.abspath(__file__))) config = ConfigParser() config.read('config.ini') @@ -68,9 +73,6 @@ template_reverse = ("$TTL 2D\n" "{ptr_records}\n") -for arg in sys.argv: - if arg=="--force": - write_dns_files(api_client) def write_dns_files(api_client): for zone in api_client.list("dns/zones"): @@ -174,7 +176,7 @@ def write_dns_files(api_client): aaaa_records=aaaa_records, cname_records=cname_records) - filename = 'generated/dns.{zone}.zone'.format(zone=zone_name) + filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) @@ -272,7 +274,7 @@ def write_dns_reverse_file(api_client): mx_records=mx_records, ptr_records = ptr_records) - filename = 'generated/dns.{zone}.zone'.format(zone=zone_name) + filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) @@ -309,7 +311,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'generated/dns.{zone}.zone'.format(zone=zone6_name) + filename = path+'/generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'a') as f: f.write(zone_file_content) else: @@ -320,7 +322,7 @@ def write_dns_reverse_file(api_client): ptr_records = ptr_records) - filename = 'generated/dns.{zone}.zone'.format(zone=zone6_name) + filename = path+'/generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'w+') as f: f.write(zone_file_content) zone_v6.append(zone6_name) @@ -329,6 +331,10 @@ api_client = Re2oAPIClient(api_hostname, api_username, api_password) client_hostname = socket.gethostname().split('.', 1)[0] +for arg in sys.argv: + if arg=="--force": + write_dns_files(api_client) + for service in api_client.list("services/regen/"): if service['hostname'] == client_hostname and \ service['service_name'] == 'dns' and \ From f4ea98921b911867740331af93495e40a048e229 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Thu, 9 Aug 2018 15:03:52 +0200 Subject: [PATCH 21/45] abs path dns --- main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index a80d893..2b495f0 100755 --- a/main.py +++ b/main.py @@ -8,13 +8,12 @@ from re2oapi import Re2oAPIClient import sys import os -import os path =(os.path.dirname(os.path.abspath(__file__))) config = ConfigParser() -config.read('config.ini') +config.read(path+'/config.ini') api_hostname = config.get('Re2o', 'hostname') api_password = config.get('Re2o', 'password') From 8d06dbd35dbb7bfd29a9587445b917b94e82c4de Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Thu, 9 Aug 2018 15:09:35 +0200 Subject: [PATCH 22/45] no tls on adm --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 2b495f0..eb4ce59 100755 --- a/main.py +++ b/main.py @@ -326,7 +326,7 @@ def write_dns_reverse_file(api_client): f.write(zone_file_content) zone_v6.append(zone6_name) -api_client = Re2oAPIClient(api_hostname, api_username, api_password) +api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=True) client_hostname = socket.gethostname().split('.', 1)[0] From 9a567c8fb2390f45dea3b9b285af5af86a98b185 Mon Sep 17 00:00:00 2001 From: Charlie Jacomme Date: Thu, 9 Aug 2018 15:10:12 +0200 Subject: [PATCH 23/45] no tls on adm --- main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index eb4ce59..2fbf025 100755 --- a/main.py +++ b/main.py @@ -11,7 +11,6 @@ import os path =(os.path.dirname(os.path.abspath(__file__))) - config = ConfigParser() config.read(path+'/config.ini') @@ -326,7 +325,7 @@ def write_dns_reverse_file(api_client): f.write(zone_file_content) zone_v6.append(zone6_name) -api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=True) +api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=False) client_hostname = socket.gethostname().split('.', 1)[0] From 64ee985f7adc58fa52ad081665d81ebd9ca7e522 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Sat, 11 Aug 2018 00:52:17 +0200 Subject: [PATCH 24/45] Recharge knot --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 2fbf025..f0077a9 100755 --- a/main.py +++ b/main.py @@ -340,3 +340,4 @@ for service in api_client.list("services/regen/"): write_dns_files(api_client) write_dns_reverse_file(api_client) api_client.patch(service['api_url'], data={'need_regen': False}) + os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') From 90f9d15cedb81ca7064d47a70782cd53ec3bc696 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Sun, 12 Aug 2018 14:29:44 +0200 Subject: [PATCH 25/45] Fix des records A, AAAA et CNAME portant le nom de zones --- main.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index f0077a9..06062e2 100755 --- a/main.py +++ b/main.py @@ -73,7 +73,16 @@ template_reverse = ("$TTL 2D\n" def write_dns_files(api_client): - for zone in api_client.list("dns/zones"): + dns_zones = api_client.list("dns/zones") + zone_names = {zone['name'][1:] for zone in dns_zones} + records_to_add = {name: {'a': [], 'aaaa': [], 'cname': []} for name in zone_names} + + def add_record(zone, record, type): + nonlocal records_to_add + records_to_add[zone][type].append(record) + return False + + for zone in sorted(dns_zones, key=lambda zone:zone['name'].count('.')): zone_name = zone['name'][1:] now = datetime.datetime.now(datetime.timezone.utc) @@ -146,6 +155,7 @@ def write_dns_files(api_client): template_a.format(hostname=x['hostname'], ipv4=x['ipv4']) for x in zone['a_records'] + if (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_a.format(hostname='@', ipv4=ip['ipv4']), 'a')) ) aaaa_records = "\n".join( @@ -153,14 +163,25 @@ def write_dns_files(api_client): ipv6=ip['ipv6']) for x in zone['aaaa_records'] for ip in x['ipv6'] - if x['ipv6'] is not None + if x['ipv6'] is not None + and (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_aaaa.format(hostname='@', ipv6=ip['ipv6']), 'aaaa')) ) cname_records = "\n".join( template_cname.format(hostname=x['hostname'], alias=x['alias']) for x in zone['cname_records'] + if (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_cname.format(hostname='@', alias=x['alias']), 'cname')) ) + + if records_to_add[zone_name]['a']: + a_records += "\n" + "\n".join(records_to_add[zone_name]['a']) + + if records_to_add[zone_name]['aaaa']: + aaaa_records += "\n" + "\n".join(records_to_add[zone_name]['aaaa']) + + if records_to_add[zone_name]['cname']: + cname_records += "\n" + "\n".join(records_to_add[zone_name]['cname']) zone_file_content = template_zone.format(soa=soa, originv4=originv4, @@ -340,4 +361,6 @@ for service in api_client.list("services/regen/"): write_dns_files(api_client) write_dns_reverse_file(api_client) api_client.patch(service['api_url'], data={'need_regen': False}) - os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') + ok = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') + if not ok: + os.system('/usr/sbin/knotc zone-reload') From 4bba92b930aeea4e48762abede7a338c20897994 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 27 Aug 2018 22:29:23 +0200 Subject: [PATCH 26/45] Formatage --- main.py | 336 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 183 insertions(+), 153 deletions(-) diff --git a/main.py b/main.py index 06062e2..af58ecc 100755 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ import sys import os -path =(os.path.dirname(os.path.abspath(__file__))) +path = os.path.dirname(os.path.abspath(__file__)) config = ConfigParser() config.read(path+'/config.ini') @@ -18,14 +18,17 @@ api_hostname = config.get('Re2o', 'hostname') api_password = config.get('Re2o', 'password') api_username = config.get('Re2o', 'username') -template_soa = ("$ORIGIN {zone}.\n" - "@ IN SOA {ns}. {mail} (\n" - " {serial} ; serial\n" - " {refresh} ; refresh\n" - " {retry} ; retry\n" - " {expire} ; expire\n" - " {ttl} ; ttl\n" - ")") +template_soa = ( + "$ORIGIN {zone}.\n" + "@ IN SOA {ns}. {mail} (\n" + " {serial} ; serial\n" + " {refresh} ; refresh\n" + " {retry} ; retry\n" + " {expire} ; expire\n" + " {ttl} ; ttl\n" + ")" +) + template_originv4 = "@ IN A {ipv4}" template_originv6 = "@ IN AAAA {ipv6}" template_ns = "@ IN NS {target}." @@ -38,56 +41,49 @@ template_cname = "{hostname} IN CNAME {alias}." template_ptr = "{target} IN PTR {hostname}." template_sshfp = "{hostname} SSHFP {algo} {type} {fp}" -template_zone = ("$TTL 2D\n" - "{soa}\n" - "\n" - "{originv4}\n" - "{originv6}\n" - "\n" - "{ns_records}\n" - "\n" - "{fp_records}\n" - "\n" - "{ns_records}\n" - "{mx_records}\n" - "\n" - "{txt_records}\n" - "\n" - "{srv_records}\n" - "\n" - "{a_records}\n" - "\n" - "{aaaa_records}\n" - "\n" - "{cname_records}") +template_zone = ( + "$TTL 2D\n" + "{soa}\n" + "\n" + "{originv4}\n" + "{originv6}\n" + "\n" + "{ns_records}\n" + "\n" + "{fp_records}\n" + "\n" + "{ns_records}\n" + "{mx_records}\n" + "\n" + "{txt_records}\n" + "\n" + "{srv_records}\n" + "\n" + "{a_records}\n" + "\n" + "{aaaa_records}\n" + "\n" + "{cname_records}" +) -template_reverse = ("$TTL 2D\n" - "{soa}\n" - "\n" - "{ns_records}\n" - "\n" - "{mx_records}\n" - "\n" - "{ptr_records}\n") +template_reverse = ( + "$TTL 2D\n" + "{soa}\n" + "\n" + "{ns_records}\n" + "\n" + "{mx_records}\n" + "\n" + "{ptr_records}\n" +) - - -def write_dns_files(api_client): - dns_zones = api_client.list("dns/zones") - zone_names = {zone['name'][1:] for zone in dns_zones} - records_to_add = {name: {'a': [], 'aaaa': [], 'cname': []} for name in zone_names} - - def add_record(zone, record, type): - nonlocal records_to_add - records_to_add[zone][type].append(record) - return False - - for zone in sorted(dns_zones, key=lambda zone:zone['name'].count('.')): +def write_dns_files(api_client, processes): + for zone in api_client.list("dns/zones"): zone_name = zone['name'][1:] now = datetime.datetime.now(datetime.timezone.utc) serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) - + soa_mail_fields = zone['soa']['mail'].split('@') soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), soa_mail_fields[1]) @@ -95,105 +91,129 @@ def write_dns_files(api_client): ns = zone['ns_records'][0]['target'] else: ns = "ns."+zone_name+"." - soa = template_soa.format(zone=zone_name, - mail=soa_mail, - serial=serial, - ns=ns, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl']) - + + soa = template_soa.format( + zone=zone_name, + mail=soa_mail, + serial=serial, + ns=ns, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl'] + ) + if zone['originv4'] is not None: originv4 = template_originv4.format(ipv4=zone['originv4']['ipv4']) else: - originv4 = "" + originv4 = "" if zone['originv6'] is not None: originv6 = template_originv6.format(ipv6=zone['originv6']) else: originv6 = "" - + ns_records = "\n".join( template_ns.format(target=x['target']) for x in zone['ns_records'] ) fp_records = "\n".join( - template_sshfp.format(hostname=host['hostname'], algo=fp['algo_id'], type="1", fp=fp['hash']['1'] ) + template_sshfp.format( + hostname=host['hostname'], + algo=fp['algo_id'], + type="1", + fp=fp['hash']['1'] + ) + "\n" + - template_sshfp.format(hostname=host['hostname'], algo=fp['algo_id'], type="2", fp=fp['hash']['2'] ) + template_sshfp.format( + hostname=host['hostname'], + algo=fp['algo_id'], + type="2", + fp=fp['hash']['2'] + ) for host in zone['sshfp_records'] for fp in host['sshfp'] ) - - + mx_records = "\n".join( - template_mx.format(priority=x['priority'], - target=x['target']) + template_mx.format( + priority=x['priority'], + target=x['target'] + ) for x in zone['mx_records'] ) - + txt_records = "\n".join( - template_txt.format(field1=x['field1'], - field2=x['field2']) + template_txt.format( + field1=x['field1'], + field2=x['field2'] + ) for x in zone['txt_records'] ) - + srv_records = "\n".join( - template_srv.format(service=x['service'], - protocol=x['protocol'], - zone=zone_name, - ttl=x['ttl'], - priority=x['priority'], - weight=x['weight'], - port=x['port'], - target=x['target']) + template_srv.format( + service=x['service'], + protocol=x['protocol'], + zone=zone_name, + ttl=x['ttl'], + priority=x['priority'], + weight=x['weight'], + port=x['port'], + target=x['target'] + ) for x in zone['srv_records'] ) a_records = "\n".join( - template_a.format(hostname=x['hostname'], - ipv4=x['ipv4']) + template_a.format( + hostname=x['hostname'], + ipv4=x['ipv4'] + ) for x in zone['a_records'] - if (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_a.format(hostname='@', ipv4=ip['ipv4']), 'a')) ) aaaa_records = "\n".join( - template_aaaa.format(hostname=x['hostname'], - ipv6=ip['ipv6']) - for x in zone['aaaa_records'] + template_aaaa.format( + hostname=x['hostname'], + ipv6=ip['ipv6'] + ) + for x in zone['aaaa_records'] for ip in x['ipv6'] if x['ipv6'] is not None - and (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_aaaa.format(hostname='@', ipv6=ip['ipv6']), 'aaaa')) ) - + + aaaa_records = "\n".join( + template_aaaa.format( + hostname=x['hostname'], + ipv6=ip['ipv6'] + ) + for x in zone['aaaa_records'] + for ip in x['ipv6'] + if x['ipv6'] is not None + ) + cname_records = "\n".join( - template_cname.format(hostname=x['hostname'], - alias=x['alias']) + template_cname.format( + hostname=x['hostname'], + alias=x['alias'] + ) for x in zone['cname_records'] - if (True if x['hostname'] + '.' + zone_name not in zone_names else add_record(x['hostname'] + '.' + zone_name, template_cname.format(hostname='@', alias=x['alias']), 'cname')) ) - if records_to_add[zone_name]['a']: - a_records += "\n" + "\n".join(records_to_add[zone_name]['a']) - - if records_to_add[zone_name]['aaaa']: - aaaa_records += "\n" + "\n".join(records_to_add[zone_name]['aaaa']) - - if records_to_add[zone_name]['cname']: - cname_records += "\n" + "\n".join(records_to_add[zone_name]['cname']) - - zone_file_content = template_zone.format(soa=soa, - originv4=originv4, - originv6=originv6, - ns_records=ns_records, - fp_records=fp_records, - mx_records=mx_records, - txt_records=txt_records, - srv_records=srv_records, - a_records=a_records, - aaaa_records=aaaa_records, - cname_records=cname_records) + zone_file_content = template_zone.format( + soa=soa, + originv4=originv4, + originv6=originv6, + ns_records=ns_records, + fp_records=fp_records, + mx_records=mx_records, + txt_records=txt_records, + srv_records=srv_records, + a_records=a_records, + aaaa_records=aaaa_records, + cname_records=cname_records + ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: @@ -206,7 +226,6 @@ def get_ip_reverse(ip, prefix_length): return '.'.join(ip.reverse_dns.split('.')[:prefix_length]) - def write_dns_reverse_file(api_client): """ Generate the reverve file for each reverse zone (= IpType) For each IpType, we generate both an Ipv4 reverse and a v6. @@ -237,12 +256,13 @@ def write_dns_reverse_file(api_client): ) mx_records = "\n".join( - template_mx.format(priority=x['priority'], - target=x['target']) + template_mx.format( + priority=x['priority'], + target=x['target'] + ) for x in zone['mx_records'] ) - ### We start with the v4 # We setup the network from the cidrs of the IpType @@ -275,46 +295,54 @@ def write_dns_reverse_file(api_client): elif subnet.prefixlen == 24: zone_name,prefix_length = ('.'.join(rev_dns_a[1:]), 1) - soa = template_soa.format(zone=zone_name, - mail=soa_mail, - serial=serial, - ns=ns, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl']) + soa = template_soa.format( + zone=zone_name, + mail=soa_mail, + serial=serial, + ns=ns, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl'] + ) + ptr_records = "\n".join( - template_ptr.format(hostname=host['hostname']+extension, - target=get_ip_reverse(host['ipv4'],prefix_length)) - for host in zone['ptr_records'] if host['ipv4'] in subnet + template_ptr.format( + hostname=host['hostname']+extension, + target=get_ip_reverse(host['ipv4'],prefix_length) ) - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) + for host in zone['ptr_records'] if host['ipv4'] in subnet + ) + zone_file_content = template_reverse.format( + soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records + ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) with open(filename, 'w+') as f: f.write(zone_file_content) - ### Continue with the ipv6 reverse if zone['ptr_v6_records']: net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) net_class = max(((net.prefixlen - 1) // 4) + 1, 1) zone6_name = ".".join( - netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] - )[:-1] + netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] + )[:-1] - soa = template_soa.format(zone=zone6_name, - mail=soa_mail, - serial=serial, - ns=ns, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl']) + soa = template_soa.format( + zone=zone6_name, + mail=soa_mail, + serial=serial, + ns=ns, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl'] + ) prefix_length = int((128 - net.prefixlen)/4) ptr_records = "\n".join( @@ -324,22 +352,24 @@ def write_dns_reverse_file(api_client): ) if zone6_name in zone_v6: # we already created the file, we ignore the soa - zone_file_content = template_reverse.format(soa="", - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) - + zone_file_content = template_reverse.format( + soa="", + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records + ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'a') as f: f.write(zone_file_content) else: # we create the file from scratch - zone_file_content = template_reverse.format(soa=soa, - ns_records=ns_records, - mx_records=mx_records, - ptr_records = ptr_records) - + zone_file_content = template_reverse.format( + soa=soa, + ns_records=ns_records, + mx_records=mx_records, + ptr_records = ptr_records + ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone6_name) with open(filename, 'w+') as f: From c3ad5c3ac4dd1732c3a5d3c3736af4d6beef8738 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 27 Aug 2018 22:36:12 +0200 Subject: [PATCH 27/45] Formatage --- main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index af58ecc..0adc29f 100755 --- a/main.py +++ b/main.py @@ -249,12 +249,12 @@ def write_dns_reverse_file(api_client): ns = "ns"+extension+"." soa_mail_fields = zone['soa']['mail'].split('@') soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), - soa_mail_fields[1]) + soa_mail_fields[1]) ns_records = "\n".join( template_ns.format(target=x['target']) for x in zone['ns_records'] ) - + mx_records = "\n".join( template_mx.format( priority=x['priority'], @@ -274,9 +274,9 @@ def write_dns_reverse_file(api_client): net = netaddr.IPNetwork(net) # on fragmente les subnets # dans les tailles qui vont bien. - if net.prefixlen > 24: + if net.prefixlen > 24: subnets.extend(net.subnet(32)) - elif net.prefixlen > 16: + elif net.prefixlen > 16: subnets.extend(net.subnet(24)) elif net.prefixlen > 8: subnets.extend(net.subnet(16)) @@ -327,11 +327,10 @@ def write_dns_reverse_file(api_client): ### Continue with the ipv6 reverse if zone['ptr_v6_records']: net = netaddr.IPNetwork(zone['prefix_v6']+"/"+str(zone['prefix_v6_length'])) - net_class = max(((net.prefixlen - 1) // 4) + 1, 1) + net_class = max(((net.prefixlen - 1) // 4) + 1, 1) zone6_name = ".".join( netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:] )[:-1] - soa = template_soa.format( zone=zone6_name, From 208b4d46535278babe0e2c078dd64ac177c76e5e Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 27 Aug 2018 22:39:15 +0200 Subject: [PATCH 28/45] maj argparse --- main.py | 308 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 167 insertions(+), 141 deletions(-) diff --git a/main.py b/main.py index 0adc29f..99f6600 100755 --- a/main.py +++ b/main.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 +import argparse from configparser import ConfigParser -import socket import datetime +from multiprocessing import Pool import netaddr +import os +import socket +import sys from re2oapi import Re2oAPIClient -import sys -import os - path = os.path.dirname(os.path.abspath(__file__)) config = ConfigParser() @@ -77,149 +78,158 @@ template_reverse = ( "{ptr_records}\n" ) -def write_dns_files(api_client, processes): - for zone in api_client.list("dns/zones"): - zone_name = zone['name'][1:] - now = datetime.datetime.now(datetime.timezone.utc) - serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) +def write_dns_file(zone): + zone_name = zone['name'][1:] - soa_mail_fields = zone['soa']['mail'].split('@') - soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), - soa_mail_fields[1]) - if zone['ns_records']: - ns = zone['ns_records'][0]['target'] - else: - ns = "ns."+zone_name+"." + now = datetime.datetime.now(datetime.timezone.utc) + serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) - soa = template_soa.format( + soa_mail_fields = zone['soa']['mail'].split('@') + soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), + soa_mail_fields[1]) + if zone['ns_records']: + ns = zone['ns_records'][0]['target'] + else: + ns = "ns."+zone_name+"." + + soa = template_soa.format( + zone=zone_name, + mail=soa_mail, + serial=serial, + ns=ns, + refresh=zone['soa']['refresh'], + retry=zone['soa']['retry'], + expire=zone['soa']['expire'], + ttl=zone['soa']['ttl'] + ) + + if zone['originv4'] is not None: + originv4 = template_originv4.format(ipv4=zone['originv4']['ipv4']) + else: + originv4 = "" + if zone['originv6'] is not None: + originv6 = template_originv6.format(ipv6=zone['originv6']) + else: + originv6 = "" + + ns_records = "\n".join( + template_ns.format(target=x['target']) + for x in zone['ns_records'] + ) + + fp_records = "\n".join( + template_sshfp.format( + hostname=host['hostname'], + algo=fp['algo_id'], + type="1", + fp=fp['hash']['1'] + ) + + "\n" + + template_sshfp.format( + hostname=host['hostname'], + algo=fp['algo_id'], + type="2", + fp=fp['hash']['2'] + ) + for host in zone['sshfp_records'] + for fp in host['sshfp'] + ) + + mx_records = "\n".join( + template_mx.format( + priority=x['priority'], + target=x['target'] + ) + for x in zone['mx_records'] + ) + + txt_records = "\n".join( + template_txt.format( + field1=x['field1'], + field2=x['field2'] + ) + for x in zone['txt_records'] + ) + + srv_records = "\n".join( + template_srv.format( + service=x['service'], + protocol=x['protocol'], zone=zone_name, - mail=soa_mail, - serial=serial, - ns=ns, - refresh=zone['soa']['refresh'], - retry=zone['soa']['retry'], - expire=zone['soa']['expire'], - ttl=zone['soa']['ttl'] + ttl=x['ttl'], + priority=x['priority'], + weight=x['weight'], + port=x['port'], + target=x['target'] ) + for x in zone['srv_records'] + ) - if zone['originv4'] is not None: - originv4 = template_originv4.format(ipv4=zone['originv4']['ipv4']) - else: - originv4 = "" - if zone['originv6'] is not None: - originv6 = template_originv6.format(ipv6=zone['originv6']) - else: - originv6 = "" - - ns_records = "\n".join( - template_ns.format(target=x['target']) - for x in zone['ns_records'] + a_records = "\n".join( + template_a.format( + hostname=x['hostname'], + ipv4=x['ipv4'] ) + for x in zone['a_records'] + ) - fp_records = "\n".join( - template_sshfp.format( - hostname=host['hostname'], - algo=fp['algo_id'], - type="1", - fp=fp['hash']['1'] - ) - + "\n" + - template_sshfp.format( - hostname=host['hostname'], - algo=fp['algo_id'], - type="2", - fp=fp['hash']['2'] - ) - for host in zone['sshfp_records'] - for fp in host['sshfp'] + aaaa_records = "\n".join( + template_aaaa.format( + hostname=x['hostname'], + ipv6=ip['ipv6'] ) + for x in zone['aaaa_records'] + for ip in x['ipv6'] + if x['ipv6'] is not None + ) - mx_records = "\n".join( - template_mx.format( - priority=x['priority'], - target=x['target'] - ) - for x in zone['mx_records'] + aaaa_records = "\n".join( + template_aaaa.format( + hostname=x['hostname'], + ipv6=ip['ipv6'] ) + for x in zone['aaaa_records'] + for ip in x['ipv6'] + if x['ipv6'] is not None + ) - txt_records = "\n".join( - template_txt.format( - field1=x['field1'], - field2=x['field2'] - ) - for x in zone['txt_records'] + cname_records = "\n".join( + template_cname.format( + hostname=x['hostname'], + alias=x['alias'] ) + for x in zone['cname_records'] + ) - srv_records = "\n".join( - template_srv.format( - service=x['service'], - protocol=x['protocol'], - zone=zone_name, - ttl=x['ttl'], - priority=x['priority'], - weight=x['weight'], - port=x['port'], - target=x['target'] - ) - for x in zone['srv_records'] - ) + zone_file_content = template_zone.format( + soa=soa, + originv4=originv4, + originv6=originv6, + ns_records=ns_records, + fp_records=fp_records, + mx_records=mx_records, + txt_records=txt_records, + srv_records=srv_records, + a_records=a_records, + aaaa_records=aaaa_records, + cname_records=cname_records + ) - a_records = "\n".join( - template_a.format( - hostname=x['hostname'], - ipv4=x['ipv4'] - ) - for x in zone['a_records'] - ) + filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) + with open(filename, 'w+') as f: + f.write(zone_file_content) - aaaa_records = "\n".join( - template_aaaa.format( - hostname=x['hostname'], - ipv6=ip['ipv6'] - ) - for x in zone['aaaa_records'] - for ip in x['ipv6'] - if x['ipv6'] is not None - ) - aaaa_records = "\n".join( - template_aaaa.format( - hostname=x['hostname'], - ipv6=ip['ipv6'] - ) - for x in zone['aaaa_records'] - for ip in x['ipv6'] - if x['ipv6'] is not None - ) +def write_dns_files(api_client, processes): + if processes: + with Pool(processes) as pool: + pool.map(write_dns_file, api_client.list("dns/zones")) + else: + for zone in api_client.list("dns/zones"): + write_dns_file(zone) - cname_records = "\n".join( - template_cname.format( - hostname=x['hostname'], - alias=x['alias'] - ) - for x in zone['cname_records'] - ) - zone_file_content = template_zone.format( - soa=soa, - originv4=originv4, - originv6=originv6, - ns_records=ns_records, - fp_records=fp_records, - mx_records=mx_records, - txt_records=txt_records, - srv_records=srv_records, - a_records=a_records, - aaaa_records=aaaa_records, - cname_records=cname_records - ) - - filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) - with open(filename, 'w+') as f: - f.write(zone_file_content) - - def get_ip_reverse(ip, prefix_length): """ Truncate an ip address given a prefix length """ ip = netaddr.IPAddress(ip) @@ -375,21 +385,37 @@ def write_dns_reverse_file(api_client): f.write(zone_file_content) zone_v6.append(zone6_name) -api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=False) +api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=True) client_hostname = socket.gethostname().split('.', 1)[0] -for arg in sys.argv: - if arg=="--force": - write_dns_files(api_client) +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Générer les fichiers de zone du DNS.") + parser.add_argument('-f', '--force', '--forced', help="Forcer la régénaration des fichiers de zone.", action='store_true') + parser.add_argument('-k', '--keep', help="Ne pas changer le statut du service.", action='store_true') + parser.add_argument('-p', '--processes', help="Regénérer en utilisant n processus en parallèle (par défaut ne pas parallèliser).", metavar='n', nargs=1, type=int, default=0) + args = parser.parse_args() -for service in api_client.list("services/regen/"): - if service['hostname'] == client_hostname and \ - service['service_name'] == 'dns' and \ - service['need_regen']: - write_dns_files(api_client) + if args.force: + write_dns_files(api_client, args.processes[0]) write_dns_reverse_file(api_client) - api_client.patch(service['api_url'], data={'need_regen': False}) - ok = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') - if not ok: - os.system('/usr/sbin/knotc zone-reload') + if not args.keep: + for service in api_client.list("services/regen/"): + if service['hostname'] == client_hostname and \ + service['service_name'] == 'dns' and \ + service['need_regen']: + api_client.patch(service['api_url'], data={'need_regen': False}) + else: + for service in api_client.list("services/regen/"): + if service['hostname'] == client_hostname and \ + service['service_name'] == 'dns' and \ + service['need_regen']: + write_dns_files(api_client, args.processes[0]) + write_dns_reverse_file(api_client) + if not args.keep: + api_client.patch(service['api_url'], data={'need_regen': False}) + + error = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') + if error: + # reload again and display the error message + os.system('/usr/sbin/knotc zone-reload') From 3f8139a59a2d0d37af434e84a676744257da8161 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 27 Aug 2018 22:54:13 +0200 Subject: [PATCH 29/45] =?UTF-8?q?Fix=20argument=20par=20d=C3=A9faut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 99f6600..08679b9 100755 --- a/main.py +++ b/main.py @@ -393,7 +393,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description="Générer les fichiers de zone du DNS.") parser.add_argument('-f', '--force', '--forced', help="Forcer la régénaration des fichiers de zone.", action='store_true') parser.add_argument('-k', '--keep', help="Ne pas changer le statut du service.", action='store_true') - parser.add_argument('-p', '--processes', help="Regénérer en utilisant n processus en parallèle (par défaut ne pas parallèliser).", metavar='n', nargs=1, type=int, default=0) + parser.add_argument('-p', '--processes', help="Regénérer en utilisant n processus en parallèle (par défaut ne pas parallèliser).", metavar='n', nargs=1, type=int, default=[0]) args = parser.parse_args() if args.force: From 2abba256b298edd1e16dcfeb0fd9b3d9682a90cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Elliott=20B=C3=A9cue?= Date: Sun, 16 Sep 2018 15:34:01 +0200 Subject: [PATCH 30/45] protocol => protocole --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 08679b9..81506e4 100755 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ template_originv6 = "@ IN AAAA {ipv6}" template_ns = "@ IN NS {target}." template_mx = "@ IN MX {priority} {target}." template_txt = "{field1} IN TXT {field2}" -template_srv = "_{service}._{protocol}.{zone} {ttl} IN SRV {priority} {weight} {port} {target}" +template_srv = "_{service}._{protocole}.{zone} {ttl} IN SRV {priority} {weight} {port} {target}" template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" template_cname = "{hostname} IN CNAME {alias}." @@ -155,7 +155,7 @@ def write_dns_file(zone): srv_records = "\n".join( template_srv.format( service=x['service'], - protocol=x['protocol'], + protocole=x['protocole'], zone=zone_name, ttl=x['ttl'], priority=x['priority'], From 491e8a2da0b975baf94040f818ee6f330be3ff4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Elliott=20B=C3=A9cue?= Date: Sun, 16 Sep 2018 16:17:55 +0200 Subject: [PATCH 31/45] main.py: exiger une zone pour les srv c'est bien. La faire finir par un . c'est mieux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sinon, c'est une sous-zone et on ne peut alors pas déclarer _xmpp-client._tcp correctement. Soit on déclare _xmpp-client._tcp sans zone et alors c'est .crans.org. implicitement soit on exige l'extension (le cas ici) et alors on déclare _xmpp-client._tcp.crans.org. avec le "." à la fin! --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 81506e4..6c7e12b 100755 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ template_originv6 = "@ IN AAAA {ipv6}" template_ns = "@ IN NS {target}." template_mx = "@ IN MX {priority} {target}." template_txt = "{field1} IN TXT {field2}" -template_srv = "_{service}._{protocole}.{zone} {ttl} IN SRV {priority} {weight} {port} {target}" +template_srv = "_{service}._{protocole}.{zone}. {ttl} IN SRV {priority} {weight} {port} {target}" template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" template_cname = "{hostname} IN CNAME {alias}." From c77cc9d46a87ab2443c6c1b26127b623baef5df9 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 17 Dec 2018 05:07:35 +0100 Subject: [PATCH 32/45] Ajout d'une option no-reload --- main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 6c7e12b..21feacf 100755 --- a/main.py +++ b/main.py @@ -394,6 +394,7 @@ if __name__ == '__main__': parser.add_argument('-f', '--force', '--forced', help="Forcer la régénaration des fichiers de zone.", action='store_true') parser.add_argument('-k', '--keep', help="Ne pas changer le statut du service.", action='store_true') parser.add_argument('-p', '--processes', help="Regénérer en utilisant n processus en parallèle (par défaut ne pas parallèliser).", metavar='n', nargs=1, type=int, default=[0]) + parser.add_argument('-n', '--no-reload', help="Ne pas recharger les zones dans knot", action='store_true') args = parser.parse_args() if args.force: @@ -415,7 +416,8 @@ if __name__ == '__main__': if not args.keep: api_client.patch(service['api_url'], data={'need_regen': False}) - error = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') - if error: - # reload again and display the error message - os.system('/usr/sbin/knotc zone-reload') + if not args.no_reload: + error = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') + if error: + # reload again and display the error message + os.system('/usr/sbin/knotc zone-reload') From 3cb0231c89f55100b32e568eea5f8cc584038da0 Mon Sep 17 00:00:00 2001 From: Maxime Bombar Date: Sat, 22 Dec 2018 12:20:07 +0100 Subject: [PATCH 33/45] [DNS] gestion des DNAME par le service dns --- main.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 21feacf..9d5fa78 100755 --- a/main.py +++ b/main.py @@ -39,6 +39,7 @@ template_srv = "_{service}._{protocole}.{zone}. {ttl} IN SRV {priority} {weight} template_a = "{hostname} IN A {ipv4}" template_aaaa = "{hostname} IN AAAA {ipv6}" template_cname = "{hostname} IN CNAME {alias}." +template_dname = "@ IN DNAME {zone}." template_ptr = "{target} IN PTR {hostname}." template_sshfp = "{hostname} SSHFP {algo} {type} {fp}" @@ -64,7 +65,9 @@ template_zone = ( "\n" "{aaaa_records}\n" "\n" - "{cname_records}" + "{cname_records}\n" + "\n" + "{dname_records}\n" ) template_reverse = ( @@ -202,6 +205,13 @@ def write_dns_file(zone): for x in zone['cname_records'] ) + dname_records = "\n".join( + template_dname.format( + zone=x['zone'][1:], + ) + for x in zone['dname_records'] + ) + zone_file_content = template_zone.format( soa=soa, originv4=originv4, @@ -213,7 +223,8 @@ def write_dns_file(zone): srv_records=srv_records, a_records=a_records, aaaa_records=aaaa_records, - cname_records=cname_records + cname_records=cname_records, + dname_records=dname_records, ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) From 47680039aaea129213bb69b7ef3acfd76ad84ea9 Mon Sep 17 00:00:00 2001 From: Maxime Bombar Date: Sat, 22 Dec 2018 12:21:42 +0100 Subject: [PATCH 34/45] [DNS] Pas de tls sur adm --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 9d5fa78..c8ca082 100755 --- a/main.py +++ b/main.py @@ -396,7 +396,7 @@ def write_dns_reverse_file(api_client): f.write(zone_file_content) zone_v6.append(zone6_name) -api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=True) +api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=False) client_hostname = socket.gethostname().split('.', 1)[0] From 700100d52a91564a708080897a586b4dd9a5e878 Mon Sep 17 00:00:00 2001 From: Gabriel Detraz Date: Sun, 23 Dec 2018 18:52:23 +0100 Subject: [PATCH 35/45] =?UTF-8?q?Code=20dupliqu=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/main.py b/main.py index c8ca082..0441c6d 100755 --- a/main.py +++ b/main.py @@ -177,15 +177,6 @@ def write_dns_file(zone): for x in zone['a_records'] ) - aaaa_records = "\n".join( - template_aaaa.format( - hostname=x['hostname'], - ipv6=ip['ipv6'] - ) - for x in zone['aaaa_records'] - for ip in x['ipv6'] - if x['ipv6'] is not None - ) aaaa_records = "\n".join( template_aaaa.format( From cba3c16d5cd8e413142c3e9b3624d37025b3819f Mon Sep 17 00:00:00 2001 From: Gabriel Detraz Date: Sun, 23 Dec 2018 19:31:36 +0100 Subject: [PATCH 36/45] Les enregistrements NS sont en double; heureusement le dns est permissif --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 0441c6d..196d376 100755 --- a/main.py +++ b/main.py @@ -54,7 +54,6 @@ template_zone = ( "\n" "{fp_records}\n" "\n" - "{ns_records}\n" "{mx_records}\n" "\n" "{txt_records}\n" From 7281290057bc3d7dffaa97ae1bb7732344702bf3 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 24 Dec 2018 10:46:34 +0100 Subject: [PATCH 37/45] =?UTF-8?q?Augmente=20le=20serial=20=C3=A0=20chaque?= =?UTF-8?q?=20maj=20des=20zones?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index c8ca082..82f25de 100755 --- a/main.py +++ b/main.py @@ -2,6 +2,7 @@ import argparse from configparser import ConfigParser import datetime +import json from multiprocessing import Pool import netaddr import os @@ -81,12 +82,19 @@ template_reverse = ( "{ptr_records}\n" ) +try: + with open(path + '/serial.json') as serial_json: + serial = json.load(serial_json) +except: + serial = 1 def write_dns_file(zone): + global serial + zone_name = zone['name'][1:] now = datetime.datetime.now(datetime.timezone.utc) - serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) + #serial = now.strftime("%Y%m%d") + str(int(100*(now.hour*3600 + now.minute*60 + now.second)/86400)) soa_mail_fields = zone['soa']['mail'].split('@') soa_mail = "{}.{}.".format(soa_mail_fields[0].replace('.', '\\.'), @@ -411,6 +419,8 @@ if __name__ == '__main__': if args.force: write_dns_files(api_client, args.processes[0]) write_dns_reverse_file(api_client) + with open(path + '/serial.json', 'w') as serial_json: + json.dump(serial + 1, serial_json) if not args.keep: for service in api_client.list("services/regen/"): if service['hostname'] == client_hostname and \ @@ -418,14 +428,19 @@ if __name__ == '__main__': service['need_regen']: api_client.patch(service['api_url'], data={'need_regen': False}) else: + increase_serial = False for service in api_client.list("services/regen/"): if service['hostname'] == client_hostname and \ service['service_name'] == 'dns' and \ service['need_regen']: + increase_serial = True write_dns_files(api_client, args.processes[0]) write_dns_reverse_file(api_client) if not args.keep: api_client.patch(service['api_url'], data={'need_regen': False}) + if increase_serial: + with open(path + '/serial.json', 'w') as serial_json: + json.dump(serial + 1, serial_json) if not args.no_reload: error = os.system('/usr/sbin/knotc zone-reload >/dev/null 2>&1') From f78ce878b711ca010c407f6626a6085feb4b5ae7 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 24 Dec 2018 10:47:30 +0100 Subject: [PATCH 38/45] Ajout du DNSSEC --- dnssec_generate.py | 32 ++++++++++++++++++++++++++++++++ main.py | 13 +++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 dnssec_generate.py diff --git a/dnssec_generate.py b/dnssec_generate.py new file mode 100755 index 0000000..2bb064d --- /dev/null +++ b/dnssec_generate.py @@ -0,0 +1,32 @@ +#!/usr/bin/python3 + +import json +import os +import re +import subprocess + + +path = os.path.dirname(os.path.abspath(__file__)) +try: + with open(path + '/dnssec_domains.json') as dnssec_zones: + zones = json.load(dnssec_zones) +except: + zones = [] + +if __name__ == '__main__': + ds_records = {} + for zone in zones: + ds = subprocess.check_output(['drill', '-s', '@localhost', '-t', 'DNSKEY', zone]).decode('utf-8') + try: + m = re.search(r'id = (\d+) \(ksk\)', ds) + tag = m.group(1) + ds = ds.split('\n') + ds = ds[ds.index('; equivalent DS records for key ' + tag + ':')+2].split('\t') + except: + print('Unable to find ksk for', zone) + continue + ds[0] = ds[0][ds[0][2:].index(' ')+3:] + ds[-1:] = ds[-1].split(' ') + ds_records[zone] = {'subzone': ds[0], 'ttl': ds[1], 'id': ds[4], 'algo': ds[5], 'type': ds[6], 'fp': ds[7]} + with open('dnssec.json', 'w') as dnssec: + json.dump(ds_records, dnssec) diff --git a/main.py b/main.py index 82f25de..0cb2791 100755 --- a/main.py +++ b/main.py @@ -43,6 +43,7 @@ template_cname = "{hostname} IN CNAME {alias}." template_dname = "@ IN DNAME {zone}." template_ptr = "{target} IN PTR {hostname}." template_sshfp = "{hostname} SSHFP {algo} {type} {fp}" +template_ds = "{subzone} {ttl} IN DS {id} {algo} {type} {fp}" template_zone = ( "$TTL 2D\n" @@ -69,6 +70,8 @@ template_zone = ( "{cname_records}\n" "\n" "{dname_records}\n" + "\n" + "{ds_records}\n" ) template_reverse = ( @@ -220,6 +223,15 @@ def write_dns_file(zone): for x in zone['dname_records'] ) + if zone['name'][1:] == "crans.org": + with open(path + '/dnssec.json') as ds: + zones_ds = json.load(ds) + ds_records = "" + for zone in zones_ds: + ds_records += template_ds.format(**zones_ds[zone]) + "\n" + else: + ds_records = "\n" + zone_file_content = template_zone.format( soa=soa, originv4=originv4, @@ -233,6 +245,7 @@ def write_dns_file(zone): aaaa_records=aaaa_records, cname_records=cname_records, dname_records=dname_records, + ds_records=ds_records, ) filename = path+'/generated/dns.{zone}.zone'.format(zone=zone_name) From 84d93d61b3f6ec72eb72550d7bd524945ee06794 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Mon, 24 Dec 2018 11:59:31 +0100 Subject: [PATCH 39/45] [dnssec_generate.py] drill -> knotc --- dnssec_generate.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dnssec_generate.py b/dnssec_generate.py index 2bb064d..663a050 100755 --- a/dnssec_generate.py +++ b/dnssec_generate.py @@ -16,17 +16,19 @@ except: if __name__ == '__main__': ds_records = {} for zone in zones: - ds = subprocess.check_output(['drill', '-s', '@localhost', '-t', 'DNSKEY', zone]).decode('utf-8') + cds = subprocess.check_output(['knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8') + ds = {} try: - m = re.search(r'id = (\d+) \(ksk\)', ds) - tag = m.group(1) - ds = ds.split('\n') - ds = ds[ds.index('; equivalent DS records for key ' + tag + ':')+2].split('\t') + cds = cds.split(' ') + ds['subzone'] = cds[1] + ds['id'] = cds[4] + ds['algo'] = cds[5] + ds['type'] = cds[6] + ds['fp'] = cds[7] except: print('Unable to find ksk for', zone) continue - ds[0] = ds[0][ds[0][2:].index(' ')+3:] - ds[-1:] = ds[-1].split(' ') - ds_records[zone] = {'subzone': ds[0], 'ttl': ds[1], 'id': ds[4], 'algo': ds[5], 'type': ds[6], 'fp': ds[7]} + ds['ttl'] = 172800 + ds_records[zone] = ds with open('dnssec.json', 'w') as dnssec: json.dump(ds_records, dnssec) From 7c08e6c41353bb4624dd498a7ae527244745cacf Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Tue, 25 Dec 2018 13:49:46 +0100 Subject: [PATCH 40/45] [dnssec_generate.py] Use full path for knotc --- dnssec_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dnssec_generate.py b/dnssec_generate.py index 663a050..ab5844e 100755 --- a/dnssec_generate.py +++ b/dnssec_generate.py @@ -16,7 +16,7 @@ except: if __name__ == '__main__': ds_records = {} for zone in zones: - cds = subprocess.check_output(['knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8') + cds = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8') ds = {} try: cds = cds.split(' ') From 462a0bebc483c05250bf3904562bb9142dd35918 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Tue, 25 Dec 2018 21:13:23 +0100 Subject: [PATCH 41/45] Updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 33b3457..8f999df 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,10 @@ This service uses Re2o API to generate DNS zone files ## Requirements * python3 +* knot * requirements in https://gitlab.federez.net/re2o/re2oapi + +## Scripts + +* `main.py`: Generates the zone files and reloads the zones +* `dnssec_generate.py`: Generate the DS records for the zones in `dnssec_domains.json` and writes them to `dnssec.json` From 9f9e950d5c3f08e3998dff8179cb863cbf043d91 Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Tue, 25 Dec 2018 21:14:16 +0100 Subject: [PATCH 42/45] Handle multiple KSK at the same time --- dnssec_generate.py | 31 +++++++++++++++++-------------- main.py | 3 ++- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/dnssec_generate.py b/dnssec_generate.py index ab5844e..8be2e5b 100755 --- a/dnssec_generate.py +++ b/dnssec_generate.py @@ -16,19 +16,22 @@ except: if __name__ == '__main__': ds_records = {} for zone in zones: - cds = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8') - ds = {} - try: - cds = cds.split(' ') - ds['subzone'] = cds[1] - ds['id'] = cds[4] - ds['algo'] = cds[5] - ds['type'] = cds[6] - ds['fp'] = cds[7] - except: - print('Unable to find ksk for', zone) - continue - ds['ttl'] = 172800 - ds_records[zone] = ds + cdss = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8').split('\n') + for cds in cdss: + ds = {} + try: + cds = cds.split(' ') + ds['subzone'] = cds[1] + ds['id'] = cds[4] + ds['algo'] = cds[5] + ds['type'] = cds[6] + ds['fp'] = cds[7] + except: + print('Unable to find ksk for', zone) + continue + ds['ttl'] = 172800 + if not zone in ds_records: + ds_records[zone] = [] + ds_records[zone].append(ds) with open('dnssec.json', 'w') as dnssec: json.dump(ds_records, dnssec) diff --git a/main.py b/main.py index d607356..d600ca9 100755 --- a/main.py +++ b/main.py @@ -218,7 +218,8 @@ def write_dns_file(zone): zones_ds = json.load(ds) ds_records = "" for zone in zones_ds: - ds_records += template_ds.format(**zones_ds[zone]) + "\n" + for ds in zones_ds[zone]: + ds_records += template_ds.format(**ds) + "\n" else: ds_records = "\n" From 5bdcde4a02cd5faf9d0016a5429ce7c75bfe245f Mon Sep 17 00:00:00 2001 From: Maxime Bombar Date: Thu, 27 Dec 2018 12:59:39 +0100 Subject: [PATCH 43/45] [dnssec_generate] add verbose mode --- dnssec_generate.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dnssec_generate.py b/dnssec_generate.py index 8be2e5b..bbef6a5 100755 --- a/dnssec_generate.py +++ b/dnssec_generate.py @@ -4,6 +4,7 @@ import json import os import re import subprocess +import argparse path = os.path.dirname(os.path.abspath(__file__)) @@ -13,10 +14,21 @@ try: except: zones = [] + + if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Gestion de DNSSEC") + parser.add_argument('-v', '--verbose', help="Mode verbeux", action="store_true") + args = parser.parse_args() + verbose = args.verbose ds_records = {} for zone in zones: - cdss = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8').split('\n') + if verbose: + print("Getting CDS of %s:" % (zone,)) + print("/usr/sbin/knotcs zone-read %s @ CDS" % (zone,)) + cdss = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8').split('\n') + if verbose: + print("CDS of %s = %s" % (zone, cdss)) for cds in cdss: ds = {} try: @@ -33,5 +45,8 @@ if __name__ == '__main__': if not zone in ds_records: ds_records[zone] = [] ds_records[zone].append(ds) + if verbose: + print("DS record of %s : %s" % (zone, ds)) + print("\n\n") with open('dnssec.json', 'w') as dnssec: json.dump(ds_records, dnssec) From 0bcb71e5f1b45a5a774a667737fb03bc8403ab8d Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Sat, 29 Dec 2018 13:35:18 +0100 Subject: [PATCH 44/45] =?UTF-8?q?[main.py]=20Int=C3=A8gre=20la=20gestion?= =?UTF-8?q?=20du=20DNSSEC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dnssec_generate.py | 52 ---------------------------------------------- knot.py | 35 +++++++++++++++++++++++++++++++ main.py | 30 +++++++++++++++----------- 3 files changed, 53 insertions(+), 64 deletions(-) delete mode 100755 dnssec_generate.py create mode 100755 knot.py diff --git a/dnssec_generate.py b/dnssec_generate.py deleted file mode 100755 index bbef6a5..0000000 --- a/dnssec_generate.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/python3 - -import json -import os -import re -import subprocess -import argparse - - -path = os.path.dirname(os.path.abspath(__file__)) -try: - with open(path + '/dnssec_domains.json') as dnssec_zones: - zones = json.load(dnssec_zones) -except: - zones = [] - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Gestion de DNSSEC") - parser.add_argument('-v', '--verbose', help="Mode verbeux", action="store_true") - args = parser.parse_args() - verbose = args.verbose - ds_records = {} - for zone in zones: - if verbose: - print("Getting CDS of %s:" % (zone,)) - print("/usr/sbin/knotcs zone-read %s @ CDS" % (zone,)) - cdss = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8').split('\n') - if verbose: - print("CDS of %s = %s" % (zone, cdss)) - for cds in cdss: - ds = {} - try: - cds = cds.split(' ') - ds['subzone'] = cds[1] - ds['id'] = cds[4] - ds['algo'] = cds[5] - ds['type'] = cds[6] - ds['fp'] = cds[7] - except: - print('Unable to find ksk for', zone) - continue - ds['ttl'] = 172800 - if not zone in ds_records: - ds_records[zone] = [] - ds_records[zone].append(ds) - if verbose: - print("DS record of %s : %s" % (zone, ds)) - print("\n\n") - with open('dnssec.json', 'w') as dnssec: - json.dump(ds_records, dnssec) diff --git a/knot.py b/knot.py new file mode 100755 index 0000000..827df87 --- /dev/null +++ b/knot.py @@ -0,0 +1,35 @@ +import json +import os +import subprocess + + +def get_ds(zone, verbose=False): + if verbose: + print("Getting CDS of %s:" % (zone,)) + print("/usr/sbin/knotc zone-read %s @ CDS" % (zone,)) + try: + cdss = subprocess.check_output(['/usr/sbin/knotc', 'zone-read', zone, '@', 'CDS'])[:-1].decode('utf-8').split('\n') + except subprocess.CalledProcessError: + return [] + dss = [] + if verbose: + print("CDS of %s = %s" % (zone, cdss)) + for cds in cdss: + ds = {} + try: + cds = cds.split(' ') + ds['subzone'] = cds[1] + ds['id'] = cds[4] + ds['algo'] = cds[5] + ds['type'] = cds[6] + ds['fp'] = cds[7] + except: + if verbose: + print('Unable to find ksk for', zone) + continue + ds['ttl'] = 172800 + if verbose: + print("DS record of %s : %s" % (zone, ds)) + print("\n\n") + dss.append(ds) + return dss diff --git a/main.py b/main.py index d600ca9..a17f1ff 100755 --- a/main.py +++ b/main.py @@ -11,6 +11,8 @@ import sys from re2oapi import Re2oAPIClient +import knot + path = os.path.dirname(os.path.abspath(__file__)) config = ConfigParser() @@ -90,7 +92,9 @@ try: except: serial = 1 -def write_dns_file(zone): +zone_names = [] + +def write_dns_file(zone, verbose=False): global serial zone_name = zone['name'][1:] @@ -214,11 +218,9 @@ def write_dns_file(zone): ) if zone['name'][1:] == "crans.org": - with open(path + '/dnssec.json') as ds: - zones_ds = json.load(ds) ds_records = "" - for zone in zones_ds: - for ds in zones_ds[zone]: + for extension in filter(lambda zone: zone.endswith('.crans.org'), zone_names): + for ds in knot.get_ds(extension, verbose): ds_records += template_ds.format(**ds) + "\n" else: ds_records = "\n" @@ -244,13 +246,16 @@ def write_dns_file(zone): f.write(zone_file_content) -def write_dns_files(api_client, processes): +def write_dns_files(api_client, processes, verbose=False): + global zone_names + zones = api_client.list("dns/zones") + zone_names = [zone["name"][1:] for zone in zones] if processes: with Pool(processes) as pool: - pool.map(write_dns_file, api_client.list("dns/zones")) + pool.map(write_dns_file, zones) else: - for zone in api_client.list("dns/zones"): - write_dns_file(zone) + for zone in zones: + write_dns_file(zone, verbose) def get_ip_reverse(ip, prefix_length): @@ -417,11 +422,12 @@ if __name__ == '__main__': parser.add_argument('-f', '--force', '--forced', help="Forcer la régénaration des fichiers de zone.", action='store_true') parser.add_argument('-k', '--keep', help="Ne pas changer le statut du service.", action='store_true') parser.add_argument('-p', '--processes', help="Regénérer en utilisant n processus en parallèle (par défaut ne pas parallèliser).", metavar='n', nargs=1, type=int, default=[0]) - parser.add_argument('-n', '--no-reload', help="Ne pas recharger les zones dans knot", action='store_true') + parser.add_argument('-n', '--no-reload', help="Ne pas recharger les zones dans knot.", action='store_true') + parser.add_argument('-v', '--verbose', help="Afficher des informations de debug.", action='store_true') args = parser.parse_args() if args.force: - write_dns_files(api_client, args.processes[0]) + write_dns_files(api_client, args.processes[0], args.verbose) write_dns_reverse_file(api_client) with open(path + '/serial.json', 'w') as serial_json: json.dump(serial + 1, serial_json) @@ -438,7 +444,7 @@ if __name__ == '__main__': service['service_name'] == 'dns' and \ service['need_regen']: increase_serial = True - write_dns_files(api_client, args.processes[0]) + write_dns_files(api_client, args.processes[0], args.verbose) write_dns_reverse_file(api_client) if not args.keep: api_client.patch(service['api_url'], data={'need_regen': False}) From 3ab9f1873d303a54c2f290c907ff1f9644a1143b Mon Sep 17 00:00:00 2001 From: Benjamin Graillot Date: Sat, 29 Dec 2018 13:51:14 +0100 Subject: [PATCH 45/45] [knot.py] Suppression imports inutiles --- knot.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/knot.py b/knot.py index 827df87..bdc48c6 100755 --- a/knot.py +++ b/knot.py @@ -1,5 +1,3 @@ -import json -import os import subprocess