crans/scripts
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[comptage_upload] Nouveau (bébé) scripts de statistiques

Browse source
This commit is contained in:
Pierre-Elliott Bécue 2014-06-22 23:25:10 +02:00
parent b4109e55fe
commit c7a62df6f6
2 changed files with 221 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
gestion/config/upload.py
View file

@ -8,10 +8,10 @@ exempt = [ ['138.231.136.0/21', '138.231.0.0/16'],
['138.231.148.0/22', '138.231.0.0/16'] ]
#: limite soft
soft = 300
soft = 300 # Mio/24h glissantes
#: limite hard
hard = 3789
hard = 3789 # Mio/24h glissantes
#: max déconnexions
max_decos = 7
@ -23,3 +23,7 @@ disconnect_mail_hard = True
#: expéditeur des mails de déconnexion
expediteur = "disconnect@crans.org"
#: Seuil pour report dans statistiques.py
stats_upload_seuil = 300 * 1024 * 1024 # Mio
pretty_seuil = "%s Mio" % (stats_upload_seuil/1024/1024,)

215
surveillance/statistiques2.py Executable file
View file

@ -0,0 +1,215 @@
#!/bin/bash /usr/scripts/python.sh
# -*- encoding: utf-8 -*-
#
# Script qui calcule les statistiques d'upload des diverses machines
# Crans.
# Auteur : Pierre-Elliott Bécue
# Licence : GPLv3
"""
Script gérant les diverses statistiques liées au comptage
d'upload. Exécuté quotidiennement sur le serveur qui compte
l'upload.
"""
import sys
import argparse
import socket
import time
import gestion.affichage as affichage
import lc_ldap.shortcuts
import psycopg2
import psycopg2.extras
import smtplib
import gestion.config.upload as upload_config
import gestion.config as config
ldap = lc_ldap.shortcuts.lc_ldap_readonly()
encoding = "UTF-8"
pgsql = psycopg2.connect(database='filtrage', user='crans')
pgsql.set_session(autocommit=True)
curseur = pgsql.cursor(cursor_factory=psycopg2.extras.DictCursor)
def ipv4or6(addr):
"""Méthode affichang IPv4/6 en fonction du
type d'IP. C'est juste un bulk pour éviter d'utiliser
des bazookas style netaddr.
"""
if ':' in addr:
return "IPv6"
else:
return "IPv4"
def upload_srv_stats(args):
"""Méthode générant les stats d'upload des serveurs à partir
des données de la base upload.
"""
upload_srv_req = """
WITH
machines_sans_doublon
AS
(
SELECT DISTINCT ON(mac_addr)
*
FROM
machines
)
SELECT
SUM(upload) as tot_upload,
SUM(download) as tot_download,
mac,
ip_crans
FROM (
(
SELECT
sum(bytes) as upload,
'0' as download,
mac_src as mac,
ip_src as ip_crans
FROM
upload
LEFT JOIN
machines_sans_doublon
ON
machines_sans_doublon.mac_addr = upload.mac_src
WHERE
machines_sans_doublon.type = 'crans'
AND NOT
ip_src <<= inet%(ipv6_local)s
AND NOT
ip_dst <<= inet%(ipv6_local)s
AND NOT
ip_dst <<= inet%(plage_ens)s
AND NOT
ip_dst <<= inet%(appt)s
AND NOT
ip_dst <<= inet%(plage_adm)s
AND NOT
ip_dst <<= inet%(plage_ipv6)s
AND
stamp_inserted >= %(begin)s
AND
stamp_inserted <= %(end)s
GROUP BY
mac,
ip_crans
)
UNION
(
SELECT
'0' as upload,
sum(bytes) as download,
mac_dst as mac,
ip_dst as ip_crans
FROM
upload
LEFT JOIN
machines_sans_doublon
ON
machines_sans_doublon.mac_addr = upload.mac_dst
WHERE
machines_sans_doublon.type = 'crans'
AND NOT
ip_src <<= inet%(ipv6_local)s
AND NOT
ip_dst <<= inet%(ipv6_local)s
AND NOT
ip_src <<= inet%(plage_ens)s
AND NOT
ip_src <<= inet%(appt)s
AND NOT
ip_src <<= inet%(plage_adm)s
AND NOT
ip_src <<= inet%(plage_ipv6)s
AND
stamp_inserted >= %(begin)s
AND
stamp_inserted <= %(end)s
GROUP BY
mac,
ip_crans
)
)
AS
famille
GROUP BY
mac,
ip_crans
ORDER BY
tot_upload
DESC
"""
fill_in_dict = {
"begin" : args.begin,
"end" : args.end,
"plage_ens" : config.plage_ens,
"ipv6_local" : 'fe80::/8',
"plage_ipv6" : config.prefix['subnet'][0],
"appt" : config.NETs['personnel-ens'][0],
"plage_adm" : config.NETs['adm'][0],
}
curseur.execute(upload_srv_req, fill_in_dict)
upload_result = curseur.fetchall()
upload_srv = [[socket.getfqdn(ligne["ip_crans"]), ipv4or6(ligne['ip_crans']), ligne['tot_upload'], ligne['tot_download']] for ligne in upload_result if ligne["tot_upload"] >= upload_config.stats_upload_seuil]
return upload_srv
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Script d'analyse d'échange de données entre un truc et un autre.", add_help=False)
parser.add_argument("-b", "--begin", help="Date de début, dans un format \"AAAA/MM/JJ HH:MM:SS\"", type=str, action="store")
parser.add_argument("-e", "--end", help="Date de fin, dans un format \"AAAA/MM/JJTHH:MM:SS\"", type=str, action="store")
parser.add_argument("-m", "--mail", help="Envoyer un mail avec les résultats.", action="store_true")
parser.add_argument("-h", "--help", help="Affiche cette aide et quitte.", action="store_true")
args = parser.parse_args()
now = time.time()
if args.help:
parser.print_help()
sys.exit(0)
if not args.begin:
args.begin = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(now - 86400))
if not args.end:
args.end = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(now))
stats_upload = upload_srv_stats(args)
entete = [u"Nom", u"Type", u"Upload", u"Download"]
longueur = ['*', 4, 13, 13]
formats = ['s', 's', 'o', 'o']
if args.mail:
styles = [None, None, None, None]
else:
styles = [None, None, "vert", "rouge"]
fetched_upload_srv = affichage.tableau(stats_upload, titre=entete, largeur=longueur, styles=styles, format=formats, width=80)
if args.mail:
expediteur = "disconnect@crans.org"
destinataire = "disconnect@crans.org"
message = """From: %(From)s
To: %(To)s
Subject: Statistiques des =?utf-8?q?derni=C3=A8res?= 24h
X-Mailer: /usr/scripts/surveillance/statistiques.py
Content-Type: text/plain; charset="utf-8"
Upload des serveurs (à partir de %(seuil_upload)s)
%(upload_srv)
--
statistiques.py
"""
corps = message % {
'From': expediteur,
'To': destinataire,
'upload_srv': fetched_upload_srv,
'seuil_upload': upload_config.pretty_seuil,
}
mail = smtplib.SMTP('localhost')
mail.sendmail(expediteur, expediteur, corps)
else:
print fetched_upload_srv