216 lines
7.3 KiB
Python
216 lines
7.3 KiB
Python
'''
|
|
|
|
Pierre Cadart
|
|
Script pour NAS sous Linux
|
|
|
|
Utilise un accès en FTP
|
|
|
|
'''
|
|
|
|
import posixpath
|
|
import re
|
|
import config
|
|
import piexel
|
|
from ftplib import FTP
|
|
import time
|
|
import file
|
|
|
|
|
|
def ftpwalk(directory, ftp):
|
|
"""
|
|
Parcours en profondeur un dossier du serveur FTP
|
|
Effectue un parcours préfixe
|
|
"""
|
|
to_visit = [directory]
|
|
while len(to_visit) > 0:
|
|
current = to_visit.pop(0)
|
|
ftp.cwd(current)
|
|
Lfiles = []
|
|
Ldirs = []
|
|
for name, prop in ftp.mlsd():
|
|
if name.startswith('.'):
|
|
continue
|
|
if prop['type'] == 'dir':
|
|
Ldirs.append(name)
|
|
to_visit.append(current+'/'+name)
|
|
elif prop['type'] == 'file':
|
|
Lfiles.append(name)
|
|
# ne construit pas la liste complète,
|
|
# mais retourne les résultats intermédiaires
|
|
yield (current, Ldirs, Lfiles)
|
|
|
|
def visit_server(domain, conf, api):
|
|
# Connection au serveur
|
|
print('connect to:', conf.get_domain(domain))
|
|
ftp = FTP(conf.get_domain(domain), user='rez', passwd='rez')
|
|
ftp.encoding = 'UTF-8'
|
|
# Initialisation des listes de mises à jour
|
|
L_missing = [] # fichiers non trouvés sur le serveur FTP
|
|
L_unreferenced = [] # fichiers non référencés dans l'API
|
|
L_moved = [] # fichiers déplacés sur le serveur FTP
|
|
# Lecture à distance des deux BDD
|
|
for directory in conf.get_movie_dir(domain):
|
|
# Visite l'arborescence de chaque dossier
|
|
Lloc = []
|
|
for path, _, files in ftpwalk(directory, ftp):
|
|
# Vérifie si le parcours du dossier est autorisé
|
|
if not any(path.startswith(p) for p in conf.get_excluded_movie_dir(domain)):
|
|
# Ajoute les fichiers correspondants aux extensions
|
|
for f in files:
|
|
if conf.is_valid_file(f):
|
|
F = file.File(path, f)
|
|
Lloc.append(F)
|
|
# Récupère les fichiers de l'api
|
|
Lapi = []
|
|
for info in api.get_files(path='ftp://'+conf.get_domain(domain)+directory, like=1):
|
|
Lapi.append(file.File(info['path'][len('ftp://'+conf.get_domain(domain)):], info['name'], api_id=info['filable_id']))
|
|
ftp.close()
|
|
|
|
# supprime les dossiers de l'api
|
|
Lapi = [f for f in Lapi if conf.is_valid_file(f.name)]
|
|
|
|
# supprime les noms avec un '+'
|
|
Lloc = [f for f in Lloc if '+' not in f.name]
|
|
|
|
# Compare avec la liste de l'api
|
|
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
|
|
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
|
|
|
|
# Fichiers déplacés localement
|
|
Lrelink = [] # liste des références à changer
|
|
for file1 in Lmissing:
|
|
for file2 in Lunref:
|
|
if file1.filename_same(file2):
|
|
Lrelink.append((file1, file2))
|
|
for fApi, fLoc in Lrelink:
|
|
Lmissing.remove(fApi)
|
|
Lunref.remove(fLoc)
|
|
print('moved:', Lrelink)
|
|
|
|
# Linke les fichiers identiques au même film
|
|
Llink = []
|
|
for file1 in Lunref:
|
|
for file2 in Lapi:
|
|
if file1.title == file2.title:
|
|
Llink.append((file1, file2.api_id))
|
|
print('D add:', file1, file2, file2.api_id)
|
|
break
|
|
print('doubles:', sorted(Llink, key=lambda f:str(f)))
|
|
for f, _ in Llink:
|
|
Lunref.remove(f)
|
|
|
|
# Linke les films par nom si possible
|
|
APIfilms = api.get_films()
|
|
API_alltitles = []
|
|
for f in APIfilms:
|
|
if f['title']:
|
|
t = f['title'].replace(' ','').lower()
|
|
if t not in [e[0] for e in API_alltitles]:
|
|
API_alltitles.append((t, f['id']))
|
|
if f['title_vo']:
|
|
t = f['title_vo'].replace(' ','').lower()
|
|
if t not in [e[0] for e in API_alltitles]:
|
|
API_alltitles.append((t, f['id']))
|
|
API_alltitles = [f for f in API_alltitles if len(f[0])>10]
|
|
Llink2 = []
|
|
for film in Lunref:
|
|
for title, fid in API_alltitles:
|
|
if title==film.simple_name:
|
|
Llink2.append((film, fid))
|
|
break
|
|
#print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
|
|
print('easy ref:', sorted(Llink2, key=lambda f:str(f)))
|
|
for f, _ in Llink2:
|
|
Lunref.remove(f)
|
|
|
|
print('missing:',Lmissing)
|
|
print('\n'*3)
|
|
print('unreferenced:','\n'.join(str(f) for f in Lunref))
|
|
print('\n'*3)
|
|
print('unreferenced titles:', '\n'.join([f.title for f in Lunref]))
|
|
|
|
# Put les renommages / déplacements
|
|
i = 0
|
|
for filmApi, filmLoc in Lrelink:
|
|
i += 1
|
|
print('['+str(i)+'/'+str(len(Lrelink))+']'+'relink:', filmApi.title)
|
|
try:
|
|
api.debug_print = True
|
|
api.put_file(id=filmApi.api_id, path='ftp://'+conf.get_domain(domain)+filmLoc.path, name=filmLoc.name)
|
|
api.debug_print = False
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film '+filmApi.title+' not edited')
|
|
raise Exception('end')
|
|
|
|
# Poste les ajouts de doubles
|
|
i = 0
|
|
for film, filmID in Llink:
|
|
i += 1
|
|
print('['+str(i)+'/'+str(len(Llink))+']'+'link:', film.title)
|
|
try:
|
|
api.debug_print = True
|
|
api.post_file(path='ftp://'+conf.get_domain(domain)+film.path, name=film.name, type='Film', type_id=filmID, **film.additional_info())
|
|
api.debug_print = False
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film '+film.title+' not added')
|
|
raise Exception('end')
|
|
|
|
# Poste les ajouts de doubles plus complexes
|
|
i = 0
|
|
for film, filmID in Llink2:
|
|
i += 1
|
|
print('['+str(i)+'/'+str(len(Llink2))+']'+'link2:', film.title)
|
|
try:
|
|
api.debug_print = True
|
|
api.post_file(path='ftp://'+conf.get_domain(domain)+film.path, name=film.name, type='Film', type_id=filmID, **film.additional_info())
|
|
api.debug_print = False
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film '+film.title+' not added')
|
|
raise Exception('end')
|
|
|
|
# Poste tout les films locaux
|
|
i = 0
|
|
for film in Lunref:
|
|
i += 1
|
|
print('['+str(i)+'/'+str(len(Lunref))+']'+'post:', film.title)
|
|
try:
|
|
api.debug_print = True
|
|
if film.year is not None:
|
|
resp = api.post_film(title=film.title, year=film.year)
|
|
else:
|
|
resp = api.post_film(title=film.title)
|
|
if "id" in resp:
|
|
api.post_file(path='ftp://'+conf.get_domain(domain)+film.path, name=film.name, type='Film', type_id=resp["id"], **film.additional_info())
|
|
api.debug_print = False
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film '+film.title+' not posted')
|
|
raise Exception('end')
|
|
|
|
# Marque comme broken les films référencés non présents
|
|
# TODO: màj des broken_links
|
|
'''
|
|
i = 0
|
|
for film in Lmissing:
|
|
i += 1
|
|
print('['+str(i)+'/'+len(Lmissing)+']'+'broken:', film)
|
|
time.sleep(1)
|
|
'''
|
|
print('visit finished')
|
|
|
|
def main():
|
|
conf = config.Config()
|
|
api = piexel.Piexel(conf.server, conf.app, conf.token)
|
|
for dom in conf.domains:
|
|
visit_server(dom, conf, api)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|