480 lines
18 KiB
Python
480 lines
18 KiB
Python
# coding:utf-8
|
|
'''
|
|
|
|
Pierre Cadart
|
|
Script pour NAS sous Linux
|
|
|
|
Utilise un accès en FTP
|
|
|
|
'''
|
|
|
|
import posixpath
|
|
import re
|
|
import config
|
|
import piexel
|
|
import tokenizer
|
|
import filerule
|
|
from ftplib import FTP
|
|
import time
|
|
import file
|
|
|
|
|
|
def ftpwalk(directory, ftp):
|
|
"""
|
|
Parcours en profondeur un dossier du serveur FTP
|
|
Effectue un parcours préfixe,
|
|
semblable à la fonction os.walk
|
|
"""
|
|
to_visit = [directory]
|
|
while len(to_visit) > 0:
|
|
current = to_visit.pop(0)
|
|
ftp.cwd(current)
|
|
Lfiles = []
|
|
Ldirs = []
|
|
for name, prop in ftp.mlsd():
|
|
if name.startswith('.'):
|
|
continue
|
|
if prop['type'] == 'dir':
|
|
Ldirs.append(name)
|
|
to_visit.append(current + '/' + name)
|
|
elif prop['type'] == 'file':
|
|
Lfiles.append(name)
|
|
# ne construit pas la liste complète,
|
|
# mais retourne les résultats intermédiaires
|
|
yield (current, Ldirs, Lfiles)
|
|
|
|
|
|
def visit_folder(domain, api, rules, tok):
|
|
"""
|
|
Visite un dossier sur un serveur, spécifié par <domain>,
|
|
et ajoute les fichiers qui vérifient les règles <rules>,
|
|
à l'API par l'interface <api>, tout en filtrant les tokens
|
|
trouvés dans le nom des fichiers avec l'interface <tok>
|
|
"""
|
|
# Connection au serveur
|
|
print('connect to:', domain['server'])
|
|
ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password'])
|
|
ftp.encoding = 'UTF-8'
|
|
# Initialisation des listes de mises à jour
|
|
L_missing = [] # fichiers non trouvés sur le serveur FTP
|
|
L_unreferenced = [] # fichiers non référencés dans l'API
|
|
L_moved = [] # fichiers déplacés sur le serveur FTP
|
|
# Lecture des fichiers sur le serveur FTP
|
|
Lloc = []
|
|
indexed_files = []
|
|
for path, _, files in ftpwalk(domain['path'], ftp):
|
|
# Ajoute les fichiers correspondants aux extensions
|
|
for f in files:
|
|
indexed_files.append({
|
|
"path": domain['server'] + path,
|
|
"name": f
|
|
})
|
|
match = filerule.match_rules(path + '/' + f, rules)
|
|
if match:
|
|
# print('got match:',match[1], 'name:',path+'/'+f)
|
|
F = file.File(path, f, match[1])
|
|
F.extract_title(tok)
|
|
Lloc.append(F)
|
|
print('indexing all files :', len(indexed_files), ' files')
|
|
api.post_index(indexed_files)
|
|
ftp.close()
|
|
print('total loc for ', domain['server'] + domain['path'], ':', len(Lloc))
|
|
# Récupère les fichiers de l'api
|
|
Lapi = []
|
|
for info in api.get_files(path=domain['server'] + domain['path'], like=1, filable=1):
|
|
nfo = {}
|
|
if ('filable' not in info) or ('filable_type' not in info):
|
|
print('nfo:', info) # le fileable associé a été supprimé
|
|
else:
|
|
year = int(info['filable']['release_date'][:4])
|
|
nfo['YEAR'] = year
|
|
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'],
|
|
api_fileid=info['id'], api_fileable_type=info['filable_type'])
|
|
F.extract_title(tok)
|
|
match = filerule.match_rules(F.path + '/' + F.name, rules)
|
|
if match:
|
|
F.info = match[1]
|
|
F.info['YEAR'] = nfo['YEAR'] # may not work :-(
|
|
Lapi.append(F)
|
|
print('total api for ', domain['server'] + domain['path'], ':', len(Lapi))
|
|
|
|
# traite les films
|
|
Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film']
|
|
Lfilm_api = [f for f in Lapi if f.fileable_type == 'film']
|
|
handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok)
|
|
|
|
# traite les épisodes
|
|
Lepisode_loc = [f for f in Lloc if f.fileable_type == 'episode']
|
|
Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode']
|
|
handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok)
|
|
|
|
print('visit finished ', domain['server'] + domain['path'])
|
|
|
|
|
|
def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
|
|
"""
|
|
Utilise les listes des fichiers locaux Lfilm_loc et des fichiers de l'API Lfilm_api
|
|
pour mettre à jour la liste des films de l'api sur le domaine <domain>
|
|
"""
|
|
Lloc = Lfilm_loc
|
|
Lapi = Lfilm_api
|
|
|
|
# print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
|
|
# print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
|
|
# print('\n'*2)
|
|
# print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
|
|
|
|
# supprime les dossiers de l'api (ils ne devraient pas apparaître)
|
|
Lapi, Linvalid = filter_by(Lapi,
|
|
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
|
|
rules))
|
|
|
|
# Compare avec la liste de l'api
|
|
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
|
|
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
|
|
|
|
# Fichiers déplacés (ou copiés) localement
|
|
Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f: f.name)
|
|
for floc, fapi in Lrelink:
|
|
if fApi in Lmissing:
|
|
Lmissing.remove(fApi)
|
|
print('moved/copied:', Lrelink)
|
|
|
|
# Linke les fichiers du même titre (simplifié) au même film
|
|
Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f: f.title)
|
|
for f, fApi in Llink:
|
|
if fApi in Lmissing:
|
|
Lmissing.remove(fApi)
|
|
print('doubles:', sorted(Llink, key=lambda f: str(f[0])))
|
|
|
|
# Linke les films par nom si possible
|
|
APIfilms = api.get_films()
|
|
API_alltitles = []
|
|
for f in APIfilms:
|
|
if f['title']:
|
|
t = f['title'].replace(' ', '').lower()
|
|
if len(t) > 2:
|
|
if t not in [e[0] for e in API_alltitles]:
|
|
API_alltitles.append((t, f['id']))
|
|
if f['title_vo']:
|
|
t = f['title_vo'].replace(' ', '').lower()
|
|
if len(t) > 2:
|
|
if t not in [e[0] for e in API_alltitles]:
|
|
API_alltitles.append((t, f['id']))
|
|
Llink2 = []
|
|
for film in Lunref:
|
|
for title, fid in API_alltitles:
|
|
if title == film.title:
|
|
Llink2.append((film, fid))
|
|
break # pour ne pas référencer deux fois le même fichier
|
|
# print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
|
|
|
|
print('easy ref:', sorted(Llink2, key=lambda f: str(f)))
|
|
for f, _ in Llink2:
|
|
Lunref.remove(f)
|
|
|
|
print('invalid:' + '\n'.join(str(f.api_fileid) + ' ' + str(f) for f in Linvalid))
|
|
print('missing (', len(Lmissing), '):', '\n'.join(
|
|
[str(e.api_id) + ':' + repr(e) + '(' + e.title + ')' for e in sorted(Lmissing, key=lambda e: e.title)]))
|
|
print('unreferenced:' + '\n'.join(str(f) for f in sorted(Lunref, key=lambda e: e.title)))
|
|
# print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
|
|
|
|
return
|
|
|
|
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
|
|
for i, film in enumerate(Linvalid):
|
|
print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', film)
|
|
try:
|
|
resp = api.delete_file(id=film.api_fileid)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + film.title + ' not deleted')
|
|
raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# Supprime les fichiers qui n'existent plus
|
|
for i, film in enumerate(Lmissing):
|
|
print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', film)
|
|
try:
|
|
resp = api.delete_file(id=film.api_fileid)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + film.title + ' not deleted')
|
|
raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# Put les renommages / déplacements
|
|
i = 0
|
|
for filmLoc, filmApi in Lrelink:
|
|
i += 1
|
|
print('[' + str(i) + '/' + str(len(Lrelink)) + ']' + 'relink:', filmApi.title)
|
|
try:
|
|
api.put_file(id=filmApi.api_fileid, path=domain['server'] + filmLoc.path, name=filmLoc.name)
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + filmApi.title + ' not edited')
|
|
raise Exception('end')
|
|
|
|
# Poste les ajouts de doubles
|
|
i = 0
|
|
for film, filmAPI in Llink:
|
|
filmID = filmAPI.api_id
|
|
i += 1
|
|
print('[' + str(i) + '/' + str(len(Llink)) + ']' + 'link:', film.title)
|
|
try:
|
|
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID)
|
|
if 'id' in resp:
|
|
post_markers(api, film, resp['id'])
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + film.title + ' not added')
|
|
raise Exception('end')
|
|
|
|
# Poste les ajouts de doubles plus complexes
|
|
i = 0
|
|
for film, filmID in Llink2:
|
|
i += 1
|
|
print('[' + str(i) + '/' + str(len(Llink2)) + ']' + 'link2:', film.title)
|
|
try:
|
|
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID,
|
|
**film.additional_info())
|
|
if 'id' in resp:
|
|
post_markers(api, film, resp['id'])
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + film.title + ' not added')
|
|
raise Exception('end')
|
|
|
|
# Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher)
|
|
i = 0
|
|
Lcannot_post = []
|
|
for film in Lunref:
|
|
i += 1
|
|
print('[' + str(i) + '/' + str(len(Lunref)) + ']' + 'post:', film.title, str(film.info.get('YEAR')))
|
|
try:
|
|
posted = False
|
|
if 'YEAR' in film.info: # tente avec l'année spécifié en premier
|
|
resp = api.post_film(title=film.title, year=film.info['YEAR'])
|
|
if "id" in resp: # id du film
|
|
posted = True
|
|
if not posted:
|
|
resp = api.post_film(title=film.title)
|
|
if "id" in resp: # id du film
|
|
posted = True
|
|
|
|
if posted:
|
|
print('post: path=', domain['server'] + film.path)
|
|
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=resp["id"])
|
|
if 'id' in resp: # id du file
|
|
post_markers(api, film, resp['id'])
|
|
else:
|
|
Lcannot_post.append(film)
|
|
|
|
print('response:', resp)
|
|
time.sleep(1)
|
|
except Exception as e:
|
|
print(e)
|
|
print('film ' + film.title + ' not posted')
|
|
raise Exception('end')
|
|
|
|
# TODO: traiter les films non postés (Lcannot_post)
|
|
|
|
|
|
def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
|
|
"""
|
|
Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api
|
|
pour mettre à jour la liste des episodes et des series de l'api sur le domaine <domain>
|
|
"""
|
|
Lloc = Lepisode_loc
|
|
Lapi = Lepisode_api
|
|
|
|
# fichiers invalides
|
|
Lapi, Linvalid = filter_by(Lapi,
|
|
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
|
|
rules))
|
|
|
|
# Compare avec la liste de l'api
|
|
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
|
|
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
|
|
|
|
# de même avec les noms de séries
|
|
Lseries_loc = list(set([f.info['SERIE'] for f in Lloc]))
|
|
Lseries_api = list(set([f.info['SERIE'] for f in Lapi]))
|
|
|
|
Lunref_serie = [s for s in Lseries_loc if s not in Lseries_api]
|
|
|
|
# récupère les séries, et les correspondances des noms locaux
|
|
APIseries = api.get_series(episodes=1)
|
|
APIepisodes = api.get_episodes(files=1)
|
|
APIepisodes_byid = {e['id']: e for e in APIepisodes}
|
|
APIfiles_byid = {f.api_id: f for f in Lapi}
|
|
series_id_bytitle = {}
|
|
series_id_bysimpletitle = {}
|
|
for s in APIseries:
|
|
series_id_bysimpletitle[s['title'].lower()] = s['id']
|
|
for e_id in [e['id'] for e in s['episodes']]:
|
|
e = APIepisodes_byid[e_id]
|
|
if 'files' in e:
|
|
for f in e['files']:
|
|
serie_title = APIfiles_byid[f['id']].info['SERIE']
|
|
series_id_bytitle[serie_title] = s['id']
|
|
|
|
# tente de lier les séries par nom simplifié
|
|
for s in Lseries_loc:
|
|
if s.lower() in series_id_bysimpletitle:
|
|
series_id_bytitle[s] = series_id_bysimpletitle[s.lower()]
|
|
if s in Lunref_serie:
|
|
Lunref_serie.remove(s)
|
|
|
|
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
|
|
for i, episode in enumerate(Linvalid):
|
|
print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', episode)
|
|
try:
|
|
resp = api.delete_file(id=episode.api_fileid)
|
|
except Exception as e:
|
|
print(e)
|
|
print('episode ' + episode.title + ' not deleted')
|
|
raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# Supprime les fichiers qui n'existent plus
|
|
for i, episode in enumerate(Lmissing):
|
|
print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', episode)
|
|
try:
|
|
resp = api.delete_file(id=episode.api_fileid)
|
|
except Exception as e:
|
|
print(e)
|
|
print('episode ' + episode.title + ' not deleted')
|
|
raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# Poste les séries non présentes, récupère les références correspondantes
|
|
series_not_found = []
|
|
for i, serie in enumerate(Lunref_serie):
|
|
print('[' + str(i + 1) + '/' + str(len(Lunref_serie)) + ']' + 'unref_serie:', serie)
|
|
try:
|
|
resp = api.post_serie(title=serie)
|
|
if 'id' in resp:
|
|
series_id_bytitle[serie] = resp['id']
|
|
print('response: ', resp)
|
|
else:
|
|
series_not_found.append(serie)
|
|
print('not found: ', resp)
|
|
except Exception as e:
|
|
print(e)
|
|
print('serie ' + serie + ' not posted')
|
|
raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# filtre les épisodes, enlève les séries non référencées
|
|
Lunref = [f for f in Lunref if f.info['SERIE'] in series_id_bytitle]
|
|
|
|
# Poste les episodes locaux
|
|
for i, episode in enumerate(Lunref):
|
|
print('[' + str(i + 1) + '/' + str(len(Lunref)) + ']' + 'post:', episode)
|
|
try:
|
|
serie_id = series_id_bytitle[episode.info['SERIE']]
|
|
episode_num = int(episode.info['EPISODE_NUM'])
|
|
season_num = int(episode.info['SEASON_NUM'])
|
|
resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num)
|
|
if 'id' in resp:
|
|
resp = api.post_file(path=domain['server'] + episode.path, name=episode.name, type='episode',
|
|
type_id=resp["id"])
|
|
print('response: ', resp)
|
|
else:
|
|
print('episode not posted:', resp)
|
|
except Exception as e:
|
|
print(e)
|
|
print('episode ' + episode.title + ' not posted')
|
|
# raise Exception('end')
|
|
time.sleep(1)
|
|
|
|
# TODO: traiter les séries non référencées (series_not_found)
|
|
|
|
|
|
def filter_by(L, f_prop):
|
|
"""
|
|
Sépare la liste L en deux listes,
|
|
suivant s'ils vérifient la propriété ou non
|
|
"""
|
|
prop_true, prop_false = [], []
|
|
for e in L:
|
|
if f_prop(e):
|
|
prop_true.append(e)
|
|
else:
|
|
prop_false.append(e)
|
|
return prop_true, prop_false
|
|
|
|
|
|
def update_find_by_common(L1, L2, f_prop):
|
|
"""
|
|
Effectue une recherche de propriétés commune entre les éléments
|
|
de L1 et L2, et retourne les listes modifiés:
|
|
L1, L2, Lassociation
|
|
"""
|
|
found = find_by_common(L1, L2, f_prop)
|
|
found1 = [f[0] for f in found]
|
|
found2 = [f[1] for f in found]
|
|
L1 = [e for e in L1 if e not in found1]
|
|
L2 = [e for e in L2 if e not in found2]
|
|
return L1, L2, found
|
|
|
|
|
|
def find_by_common(L1, L2, f_prop):
|
|
"""
|
|
Associe les éléments de <L1> et <L2> à travers leur propriétés
|
|
trouvées avec la fonction f_prop.
|
|
Les éléments de L1 ne sont associés qu'une fois, ceux de L2 plusieurs
|
|
"""
|
|
found = []
|
|
Lprop2 = [f_prop(e) for e in L2]
|
|
for i, e1 in enumerate(L1):
|
|
prop1 = f_prop(e1)
|
|
if prop1 in Lprop2:
|
|
j = Lprop2.find(prop1)
|
|
e2 = L2[j]
|
|
found.append((e1, e2))
|
|
break
|
|
return found
|
|
|
|
|
|
def post_markers(api, file_, fileid):
|
|
"""
|
|
Poste sur l'api par l'interface <api> tout les marqueurs
|
|
du fichier <file_>, indexé à l'ID <fileid>
|
|
"""
|
|
for lang in file_.markers['lang']:
|
|
api.post_file_language(fileid, value=lang)
|
|
time.sleep(1)
|
|
for qual in file_.markers['quality']:
|
|
api.post_file_qualities(fileid, value=qual)
|
|
time.sleep(1)
|
|
for sub in file_.markers['subtitle']:
|
|
api.post_file_subtitle(fileid, value=sub)
|
|
time.sleep(1)
|
|
|
|
|
|
def main():
|
|
"""
|
|
Fonction principale du programme, réalise toute les opérations
|
|
de chargement des configurations, d'initialisation des interfaces,
|
|
et parcours tout les dossiers cherchables.
|
|
"""
|
|
conf = config.Config()
|
|
api = piexel.Piexel(conf.server, conf.app, conf.token)
|
|
tokens = tokenizer.Tokenizer(conf, api)
|
|
|
|
folders = api.get_folders()
|
|
rules = api.get_paths()
|
|
|
|
for fold in folders[1:]:
|
|
applicable = [filerule.FileRule(re.escape(fold['path']) + '\\/' + r['regex'], conf) for r in rules if
|
|
int(r['indexer_folder_id']) == fold['id']]
|
|
visit_folder(fold, api, applicable, tokens)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|