add indexed files

This commit is contained in:
bdvllrs 2018-02-06 14:31:01 +01:00
parent cbbb8148b5
commit bf1fa8e3dd
7 changed files with 250 additions and 190 deletions

183
main.py
View file

@ -1,4 +1,4 @@
#coding:utf-8
# coding:utf-8
'''
Pierre Cadart
@ -36,12 +36,13 @@ def ftpwalk(directory, ftp):
continue
if prop['type'] == 'dir':
Ldirs.append(name)
to_visit.append(current+'/'+name)
to_visit.append(current + '/' + name)
elif prop['type'] == 'file':
Lfiles.append(name)
# ne construit pas la liste complète,
# mais retourne les résultats intermédiaires
yield (current, Ldirs, Lfiles)
yield (current, Ldirs, Lfiles)
def visit_folder(domain, api, rules, tok):
"""
@ -55,39 +56,47 @@ def visit_folder(domain, api, rules, tok):
ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password'])
ftp.encoding = 'UTF-8'
# Initialisation des listes de mises à jour
L_missing = [] # fichiers non trouvés sur le serveur FTP
L_unreferenced = [] # fichiers non référencés dans l'API
L_moved = [] # fichiers déplacés sur le serveur FTP
L_missing = [] # fichiers non trouvés sur le serveur FTP
L_unreferenced = [] # fichiers non référencés dans l'API
L_moved = [] # fichiers déplacés sur le serveur FTP
# Lecture des fichiers sur le serveur FTP
Lloc = []
indexed_files = []
for path, _, files in ftpwalk(domain['path'], ftp):
# Ajoute les fichiers correspondants aux extensions
for f in files:
match = filerule.match_rules(path+'/'+f, rules)
indexed_files.append({
"path": domain['server'] + path,
"name": f
})
match = filerule.match_rules(path + '/' + f, rules)
if match:
#print('got match:',match[1], 'name:',path+'/'+f)
# print('got match:',match[1], 'name:',path+'/'+f)
F = file.File(path, f, match[1])
F.extract_title(tok)
Lloc.append(F)
print('indexing all files :', len(indexed_files), ' files')
api.post_index(indexed_files)
ftp.close()
print('total loc for ',domain['server']+domain['path'],':', len(Lloc))
print('total loc for ', domain['server'] + domain['path'], ':', len(Lloc))
# Récupère les fichiers de l'api
Lapi = []
for info in api.get_files(path=domain['server']+domain['path'], like=1, filable=1):
for info in api.get_files(path=domain['server'] + domain['path'], like=1, filable=1):
nfo = {}
if ('filable' not in info) or ('filable_type' not in info):
print('nfo:', info) # le fileable associé a été supprimé
print('nfo:', info) # le fileable associé a été supprimé
else:
year = int(info['filable']['release_date'][:4])
nfo['YEAR'] = year
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'], api_fileid=info['id'], api_fileable_type=info['filable_type'])
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'],
api_fileid=info['id'], api_fileable_type=info['filable_type'])
F.extract_title(tok)
match = filerule.match_rules(F.path+'/'+F.name, rules)
match = filerule.match_rules(F.path + '/' + F.name, rules)
if match:
F.info = match[1]
F.info['YEAR'] = nfo['YEAR'] # may not work :-(
F.info['YEAR'] = nfo['YEAR'] # may not work :-(
Lapi.append(F)
print('total api for ',domain['server']+domain['path'],':', len(Lapi))
print('total api for ', domain['server'] + domain['path'], ':', len(Lapi))
# traite les films
Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film']
@ -99,7 +108,8 @@ def visit_folder(domain, api, rules, tok):
Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode']
handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok)
print('visit finished ',domain['server']+domain['path'])
print('visit finished ', domain['server'] + domain['path'])
def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
"""
@ -109,84 +119,87 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
Lloc = Lfilm_loc
Lapi = Lfilm_api
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('\n'*2)
#print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
# print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
# print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
# print('\n'*2)
# print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
# supprime les dossiers de l'api (ils ne devraient pas apparaître)
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules))
Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
# Fichiers déplacés (ou copiés) localement
Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f:f.name)
Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f: f.name)
for floc, fapi in Lrelink:
if fApi in Lmissing:
Lmissing.remove(fApi)
print('moved/copied:', Lrelink)
# Linke les fichiers du même titre (simplifié) au même film
Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f:f.title)
Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f: f.title)
for f, fApi in Llink:
if fApi in Lmissing:
Lmissing.remove(fApi)
print('doubles:', sorted(Llink, key=lambda f:str(f[0])))
print('doubles:', sorted(Llink, key=lambda f: str(f[0])))
# Linke les films par nom si possible
APIfilms = api.get_films()
API_alltitles = []
for f in APIfilms:
if f['title']:
t = f['title'].replace(' ','').lower()
t = f['title'].replace(' ', '').lower()
if len(t) > 2:
if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id']))
if f['title_vo']:
t = f['title_vo'].replace(' ','').lower()
t = f['title_vo'].replace(' ', '').lower()
if len(t) > 2:
if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id']))
Llink2 = []
for film in Lunref:
for title, fid in API_alltitles:
if title==film.title:
if title == film.title:
Llink2.append((film, fid))
break # pour ne pas référencer deux fois le même fichier
#print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
print('easy ref:', sorted(Llink2, key=lambda f:str(f)))
break # pour ne pas référencer deux fois le même fichier
# print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
print('easy ref:', sorted(Llink2, key=lambda f: str(f)))
for f, _ in Llink2:
Lunref.remove(f)
print('invalid:'+'\n'.join(str(f.api_fileid)+' '+str(f) for f in Linvalid))
print('missing (', len(Lmissing), '):','\n'.join([str(e.api_id)+':'+repr(e)+'('+e.title+')' for e in sorted(Lmissing, key=lambda e:e.title)]))
print('unreferenced:'+'\n'.join(str(f) for f in sorted(Lunref, key=lambda e:e.title)))
#print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
print('invalid:' + '\n'.join(str(f.api_fileid) + ' ' + str(f) for f in Linvalid))
print('missing (', len(Lmissing), '):', '\n'.join(
[str(e.api_id) + ':' + repr(e) + '(' + e.title + ')' for e in sorted(Lmissing, key=lambda e: e.title)]))
print('unreferenced:' + '\n'.join(str(f) for f in sorted(Lunref, key=lambda e: e.title)))
# print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
return
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, film in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', film)
print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', film)
try:
resp = api.delete_file(id=film.api_fileid)
except Exception as e:
print(e)
print('film '+film.title+' not deleted')
print('film ' + film.title + ' not deleted')
raise Exception('end')
time.sleep(1)
# Supprime les fichiers qui n'existent plus
for i, film in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', film)
print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', film)
try:
resp = api.delete_file(id=film.api_fileid)
except Exception as e:
print(e)
print('film '+film.title+' not deleted')
print('film ' + film.title + ' not deleted')
raise Exception('end')
time.sleep(1)
@ -194,67 +207,68 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
i = 0
for filmLoc, filmApi in Lrelink:
i += 1
print('['+str(i)+'/'+str(len(Lrelink))+']'+'relink:', filmApi.title)
print('[' + str(i) + '/' + str(len(Lrelink)) + ']' + 'relink:', filmApi.title)
try:
api.put_file(id=filmApi.api_fileid, path=domain['server']+filmLoc.path, name=filmLoc.name)
api.put_file(id=filmApi.api_fileid, path=domain['server'] + filmLoc.path, name=filmLoc.name)
time.sleep(1)
except Exception as e:
print(e)
print('film '+filmApi.title+' not edited')
print('film ' + filmApi.title + ' not edited')
raise Exception('end')
# Poste les ajouts de doubles
i = 0
for film, filmAPI in Llink:
filmID = filmAPI.api_id
i += 1
print('['+str(i)+'/'+str(len(Llink))+']'+'link:', film.title)
print('[' + str(i) + '/' + str(len(Llink)) + ']' + 'link:', film.title)
try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID)
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID)
if 'id' in resp:
post_markers(api, film, resp['id'])
time.sleep(1)
except Exception as e:
print(e)
print('film '+film.title+' not added')
print('film ' + film.title + ' not added')
raise Exception('end')
# Poste les ajouts de doubles plus complexes
i = 0
for film, filmID in Llink2:
i += 1
print('['+str(i)+'/'+str(len(Llink2))+']'+'link2:', film.title)
print('[' + str(i) + '/' + str(len(Llink2)) + ']' + 'link2:', film.title)
try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID, **film.additional_info())
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID,
**film.additional_info())
if 'id' in resp:
post_markers(api, film, resp['id'])
time.sleep(1)
except Exception as e:
print(e)
print('film '+film.title+' not added')
print('film ' + film.title + ' not added')
raise Exception('end')
# Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher)
i = 0
Lcannot_post = []
for film in Lunref:
i += 1
print('['+str(i)+'/'+str(len(Lunref))+']'+'post:', film.title, str(film.info.get('YEAR')))
print('[' + str(i) + '/' + str(len(Lunref)) + ']' + 'post:', film.title, str(film.info.get('YEAR')))
try:
posted = False
if 'YEAR' in film.info: # tente avec l'année spécifié en premier
if 'YEAR' in film.info: # tente avec l'année spécifié en premier
resp = api.post_film(title=film.title, year=film.info['YEAR'])
if "id" in resp: # id du film
if "id" in resp: # id du film
posted = True
if not posted:
resp = api.post_film(title=film.title)
if "id" in resp: # id du film
if "id" in resp: # id du film
posted = True
if posted:
print('post: path=',domain['server']+film.path)
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=resp["id"])
if 'id' in resp: # id du file
print('post: path=', domain['server'] + film.path)
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=resp["id"])
if 'id' in resp: # id du file
post_markers(api, film, resp['id'])
else:
Lcannot_post.append(film)
@ -263,11 +277,12 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
time.sleep(1)
except Exception as e:
print(e)
print('film '+film.title+' not posted')
print('film ' + film.title + ' not posted')
raise Exception('end')
# TODO: traiter les films non postés (Lcannot_post)
def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
"""
Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api
@ -277,11 +292,13 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
Lapi = Lepisode_api
# fichiers invalides
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules))
Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
# de même avec les noms de séries
Lseries_loc = list(set([f.info['SERIE'] for f in Lloc]))
@ -292,8 +309,8 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# récupère les séries, et les correspondances des noms locaux
APIseries = api.get_series(episodes=1)
APIepisodes = api.get_episodes(files=1)
APIepisodes_byid = {e['id']:e for e in APIepisodes}
APIfiles_byid = {f.api_id:f for f in Lapi}
APIepisodes_byid = {e['id']: e for e in APIepisodes}
APIfiles_byid = {f.api_id: f for f in Lapi}
series_id_bytitle = {}
series_id_bysimpletitle = {}
for s in APIseries:
@ -314,30 +331,30 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, episode in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', episode)
print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', episode)
try:
resp = api.delete_file(id=episode.api_fileid)
except Exception as e:
print(e)
print('episode '+episode.title+' not deleted')
print('episode ' + episode.title + ' not deleted')
raise Exception('end')
time.sleep(1)
# Supprime les fichiers qui n'existent plus
for i, episode in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', episode)
print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', episode)
try:
resp = api.delete_file(id=episode.api_fileid)
except Exception as e:
print(e)
print('episode '+episode.title+' not deleted')
print('episode ' + episode.title + ' not deleted')
raise Exception('end')
time.sleep(1)
# Poste les séries non présentes, récupère les références correspondantes
series_not_found = []
for i, serie in enumerate(Lunref_serie):
print('['+str(i+1)+'/'+str(len(Lunref_serie))+']'+'unref_serie:', serie)
print('[' + str(i + 1) + '/' + str(len(Lunref_serie)) + ']' + 'unref_serie:', serie)
try:
resp = api.post_serie(title=serie)
if 'id' in resp:
@ -348,7 +365,7 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
print('not found: ', resp)
except Exception as e:
print(e)
print('serie '+serie+' not posted')
print('serie ' + serie + ' not posted')
raise Exception('end')
time.sleep(1)
@ -357,25 +374,27 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Poste les episodes locaux
for i, episode in enumerate(Lunref):
print('['+str(i+1)+'/'+str(len(Lunref))+']'+'post:', episode)
print('[' + str(i + 1) + '/' + str(len(Lunref)) + ']' + 'post:', episode)
try:
serie_id = series_id_bytitle[episode.info['SERIE']]
episode_num = int(episode.info['EPISODE_NUM'])
season_num = int(episode.info['SEASON_NUM'])
resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num)
if 'id' in resp:
resp = api.post_file(path=domain['server']+episode.path, name=episode.name, type='episode', type_id=resp["id"])
print('response: ',resp)
resp = api.post_file(path=domain['server'] + episode.path, name=episode.name, type='episode',
type_id=resp["id"])
print('response: ', resp)
else:
print('episode not posted:', resp)
except Exception as e:
print(e)
print('episode '+episode.title+' not posted')
#raise Exception('end')
print('episode ' + episode.title + ' not posted')
# raise Exception('end')
time.sleep(1)
# TODO: traiter les séries non référencées (series_not_found)
def filter_by(L, f_prop):
"""
Sépare la liste L en deux listes,
@ -389,6 +408,7 @@ def filter_by(L, f_prop):
prop_false.append(e)
return prop_true, prop_false
def update_find_by_common(L1, L2, f_prop):
"""
Effectue une recherche de propriétés commune entre les éléments
@ -402,6 +422,7 @@ def update_find_by_common(L1, L2, f_prop):
L2 = [e for e in L2 if e not in found2]
return L1, L2, found
def find_by_common(L1, L2, f_prop):
"""
Associe les éléments de <L1> et <L2> à travers leur propriétés
@ -435,6 +456,7 @@ def post_markers(api, file_, fileid):
api.post_file_subtitle(fileid, value=sub)
time.sleep(1)
def main():
"""
Fonction principale du programme, réalise toute les opérations
@ -449,9 +471,10 @@ def main():
rules = api.get_paths()
for fold in folders[1:]:
applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']]
applicable = [filerule.FileRule(re.escape(fold['path']) + '\\/' + r['regex'], conf) for r in rules if
int(r['indexer_folder_id']) == fold['id']]
visit_folder(fold, api, applicable, tokens)
if __name__ == '__main__':
main()