piexel/piexel-indexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add indexed files

Browse source
This commit is contained in:
bdvllrs 2018-02-06 14:31:01 +01:00
parent cbbb8148b5
commit bf1fa8e3dd
7 changed files with 250 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -3,3 +3,4 @@ __pycache__/*
*.sublime-project *.sublime-project
*.sublime-workspace *.sublime-workspace
config.conf config.conf
.idea/*

6
config.py
View file

@ -1,4 +1,4 @@
#coding:utf-8 # coding:utf-8
from configobj import ConfigObj from configobj import ConfigObj
import shutil import shutil
import posixpath import posixpath
@ -12,6 +12,7 @@ class Config:
""" """
Gère le fichier de configuration Gère le fichier de configuration
""" """
def __init__(self): def __init__(self):
try: try:
@ -46,9 +47,8 @@ class Config:
raise ConfigError("no 'api' section in config file") raise ConfigError("no 'api' section in config file")
for k in ['app', 'token', 'extensions', 'server']: for k in ['app', 'token', 'extensions', 'server']:
if not k in self.obj['api']: if not k in self.obj['api']:
raise ConfigError("no '"+k+"' value in config file, subsection 'api'") raise ConfigError("no '" + k + "' value in config file, subsection 'api'")
def is_valid_file(self, name): def is_valid_file(self, name):
_, ext = posixpath.splitext(name) _, ext = posixpath.splitext(name)
return ext[1:] in self.obj['api']['extensions'] return ext[1:] in self.obj['api']['extensions']

21
file.py
View file

@ -1,11 +1,13 @@
#coding:utf-8 # coding:utf-8
import posixpath import posixpath
import re import re
class File: class File:
""" """
Décrit une référence de fichier dans le disque Décrit une référence de fichier dans le disque
""" """
def __init__(self, path, name, info, api_id=None, api_fileid=None, api_fileable_type=None): def __init__(self, path, name, info, api_id=None, api_fileid=None, api_fileable_type=None):
self.path = path self.path = path
self.name = name self.name = name
@ -13,7 +15,7 @@ class File:
self.markers = {} self.markers = {}
self.api_id = api_id self.api_id = api_id
self.api_fileid = api_fileid self.api_fileid = api_fileid
self.fileable_type = {None: None, 'App\\Film':'film', 'App\\Episode':'episode'}[api_fileable_type] self.fileable_type = {None: None, 'App\\Film': 'film', 'App\\Episode': 'episode'}[api_fileable_type]
def get_ext(self): def get_ext(self):
""" """
@ -46,7 +48,7 @@ class File:
fname = fname.replace('(', ' ( ') fname = fname.replace('(', ' ( ')
fname = fname.replace(')', ' ) ') fname = fname.replace(')', ' ) ')
# 2) marqueurs # 2) marqueurs
fname, info = tok.tokenize(' '+fname+' ') fname, info = tok.tokenize(' ' + fname + ' ')
self.markers = info self.markers = info
# 3) minuscule # 3) minuscule
fname = fname.lower() fname = fname.lower()
@ -57,10 +59,10 @@ class File:
self.info['YEAR'] = m.group(1) self.info['YEAR'] = m.group(1)
if 'YEAR' in self.info: if 'YEAR' in self.info:
year = self.info['YEAR'] year = self.info['YEAR']
fname = fname.replace('['+str(year)+']', '') fname = fname.replace('[' + str(year) + ']', '')
fname = fname.replace('[ '+str(year)+' ]', '') fname = fname.replace('[ ' + str(year) + ' ]', '')
fname = fname.replace('['+str(year)+' ]', '') fname = fname.replace('[' + str(year) + ' ]', '')
fname = fname.replace('[ '+str(year)+']', '') fname = fname.replace('[ ' + str(year) + ']', '')
# 5) espaces en bout, centraux et rajoutés # 5) espaces en bout, centraux et rajoutés
fname = fname.lstrip().rstrip() fname = fname.lstrip().rstrip()
while ' ' in fname: while ' ' in fname:
@ -92,9 +94,8 @@ class File:
def __str__(self): def __str__(self):
if 'YEAR' in self.info: if 'YEAR' in self.info:
return str(self.path+'/'+self.name+' year:['+str(self.info['YEAR'])+']') return str(self.path + '/' + self.name + ' year:[' + str(self.info['YEAR']) + ']')
return str(self.path+'/'+self.name) return str(self.path + '/' + self.name)
def __repr__(self): def __repr__(self):
return str(self) return str(self)

8
filerule.py
View file

@ -2,6 +2,7 @@ import re
PATTERN_BASIC = re.compile(r'#(.+?)#') PATTERN_BASIC = re.compile(r'#(.+?)#')
def match_rules(filename, rules): def match_rules(filename, rules):
for r in rules: for r in rules:
match = r.apply(filename) match = r.apply(filename)
@ -9,11 +10,12 @@ def match_rules(filename, rules):
return r, match return r, match
return None return None
class FileRule: class FileRule:
def __init__(self, rule, conf): def __init__(self, rule, conf):
# trouve les marqueurs dans la règle # trouve les marqueurs dans la règle
self.patterns = PATTERN_BASIC.findall(rule) self.patterns = PATTERN_BASIC.findall(rule)
rg = PATTERN_BASIC.sub(lambda m:self.regFor(match=m, conf=conf), rule) rg = PATTERN_BASIC.sub(lambda m: self.regFor(match=m, conf=conf), rule)
self.rule = re.compile(rg) # on garde la version compilée (donc optimisée) self.rule = re.compile(rg) # on garde la version compilée (donc optimisée)
@staticmethod @staticmethod
@ -21,7 +23,7 @@ class FileRule:
name = match.group(0) name = match.group(0)
trunc_name = name[1:-1] trunc_name = name[1:-1]
if name == '#EXT#': if name == '#EXT#':
return '('+'|'.join(conf.extensions)+')' return '(' + '|'.join(conf.extensions) + ')'
elif name in ['#EPISODE_NUM#', '#SEASON_NUM#']: elif name in ['#EPISODE_NUM#', '#SEASON_NUM#']:
return '([0-9]+)' return '([0-9]+)'
return '([^/]+)' return '([^/]+)'
@ -34,5 +36,5 @@ class FileRule:
# trouve les différents marqueurs de règle # trouve les différents marqueurs de règle
patterns = {} patterns = {}
for i in range(len(self.patterns)): for i in range(len(self.patterns)):
patterns[self.patterns[i]] = match.group(i+1) patterns[self.patterns[i]] = match.group(i + 1)
return patterns return patterns

143
main.py
View file

@ -1,4 +1,4 @@
#coding:utf-8 # coding:utf-8
''' '''
Pierre Cadart Pierre Cadart
@ -36,13 +36,14 @@ def ftpwalk(directory, ftp):
continue continue
if prop['type'] == 'dir': if prop['type'] == 'dir':
Ldirs.append(name) Ldirs.append(name)
to_visit.append(current+'/'+name) to_visit.append(current + '/' + name)
elif prop['type'] == 'file': elif prop['type'] == 'file':
Lfiles.append(name) Lfiles.append(name)
# ne construit pas la liste complète, # ne construit pas la liste complète,
# mais retourne les résultats intermédiaires # mais retourne les résultats intermédiaires
yield (current, Ldirs, Lfiles) yield (current, Ldirs, Lfiles)
def visit_folder(domain, api, rules, tok): def visit_folder(domain, api, rules, tok):
""" """
Visite un dossier sur un serveur, spécifié par <domain>, Visite un dossier sur un serveur, spécifié par <domain>,
@ -60,34 +61,42 @@ def visit_folder(domain, api, rules, tok):
L_moved = [] # fichiers déplacés sur le serveur FTP L_moved = [] # fichiers déplacés sur le serveur FTP
# Lecture des fichiers sur le serveur FTP # Lecture des fichiers sur le serveur FTP
Lloc = [] Lloc = []
indexed_files = []
for path, _, files in ftpwalk(domain['path'], ftp): for path, _, files in ftpwalk(domain['path'], ftp):
# Ajoute les fichiers correspondants aux extensions # Ajoute les fichiers correspondants aux extensions
for f in files: for f in files:
match = filerule.match_rules(path+'/'+f, rules) indexed_files.append({
"path": domain['server'] + path,
"name": f
})
match = filerule.match_rules(path + '/' + f, rules)
if match: if match:
#print('got match:',match[1], 'name:',path+'/'+f) # print('got match:',match[1], 'name:',path+'/'+f)
F = file.File(path, f, match[1]) F = file.File(path, f, match[1])
F.extract_title(tok) F.extract_title(tok)
Lloc.append(F) Lloc.append(F)
print('indexing all files :', len(indexed_files), ' files')
api.post_index(indexed_files)
ftp.close() ftp.close()
print('total loc for ',domain['server']+domain['path'],':', len(Lloc)) print('total loc for ', domain['server'] + domain['path'], ':', len(Lloc))
# Récupère les fichiers de l'api # Récupère les fichiers de l'api
Lapi = [] Lapi = []
for info in api.get_files(path=domain['server']+domain['path'], like=1, filable=1): for info in api.get_files(path=domain['server'] + domain['path'], like=1, filable=1):
nfo = {} nfo = {}
if ('filable' not in info) or ('filable_type' not in info): if ('filable' not in info) or ('filable_type' not in info):
print('nfo:', info) # le fileable associé a été supprimé print('nfo:', info) # le fileable associé a été supprimé
else: else:
year = int(info['filable']['release_date'][:4]) year = int(info['filable']['release_date'][:4])
nfo['YEAR'] = year nfo['YEAR'] = year
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'], api_fileid=info['id'], api_fileable_type=info['filable_type']) F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'],
api_fileid=info['id'], api_fileable_type=info['filable_type'])
F.extract_title(tok) F.extract_title(tok)
match = filerule.match_rules(F.path+'/'+F.name, rules) match = filerule.match_rules(F.path + '/' + F.name, rules)
if match: if match:
F.info = match[1] F.info = match[1]
F.info['YEAR'] = nfo['YEAR'] # may not work :-( F.info['YEAR'] = nfo['YEAR'] # may not work :-(
Lapi.append(F) Lapi.append(F)
print('total api for ',domain['server']+domain['path'],':', len(Lapi)) print('total api for ', domain['server'] + domain['path'], ':', len(Lapi))
# traite les films # traite les films
Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film'] Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film']
@ -99,7 +108,8 @@ def visit_folder(domain, api, rules, tok):
Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode'] Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode']
handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok) handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok)
print('visit finished ',domain['server']+domain['path']) print('visit finished ', domain['server'] + domain['path'])
def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok): def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
""" """
@ -109,84 +119,87 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
Lloc = Lfilm_loc Lloc = Lfilm_loc
Lapi = Lfilm_api Lapi = Lfilm_api
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc]))) # print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc]))) # print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('\n'*2) # print('\n'*2)
#print('api titles:', '|'.join(sorted([f.title for f in Lapi]))) # print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
# supprime les dossiers de l'api (ils ne devraient pas apparaître) # supprime les dossiers de l'api (ils ne devraient pas apparaître)
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules)) Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api # Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
# Fichiers déplacés (ou copiés) localement # Fichiers déplacés (ou copiés) localement
Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f:f.name) Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f: f.name)
for floc, fapi in Lrelink: for floc, fapi in Lrelink:
if fApi in Lmissing: if fApi in Lmissing:
Lmissing.remove(fApi) Lmissing.remove(fApi)
print('moved/copied:', Lrelink) print('moved/copied:', Lrelink)
# Linke les fichiers du même titre (simplifié) au même film # Linke les fichiers du même titre (simplifié) au même film
Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f:f.title) Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f: f.title)
for f, fApi in Llink: for f, fApi in Llink:
if fApi in Lmissing: if fApi in Lmissing:
Lmissing.remove(fApi) Lmissing.remove(fApi)
print('doubles:', sorted(Llink, key=lambda f:str(f[0]))) print('doubles:', sorted(Llink, key=lambda f: str(f[0])))
# Linke les films par nom si possible # Linke les films par nom si possible
APIfilms = api.get_films() APIfilms = api.get_films()
API_alltitles = [] API_alltitles = []
for f in APIfilms: for f in APIfilms:
if f['title']: if f['title']:
t = f['title'].replace(' ','').lower() t = f['title'].replace(' ', '').lower()
if len(t) > 2: if len(t) > 2:
if t not in [e[0] for e in API_alltitles]: if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id'])) API_alltitles.append((t, f['id']))
if f['title_vo']: if f['title_vo']:
t = f['title_vo'].replace(' ','').lower() t = f['title_vo'].replace(' ', '').lower()
if len(t) > 2: if len(t) > 2:
if t not in [e[0] for e in API_alltitles]: if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id'])) API_alltitles.append((t, f['id']))
Llink2 = [] Llink2 = []
for film in Lunref: for film in Lunref:
for title, fid in API_alltitles: for title, fid in API_alltitles:
if title==film.title: if title == film.title:
Llink2.append((film, fid)) Llink2.append((film, fid))
break # pour ne pas référencer deux fois le même fichier break # pour ne pas référencer deux fois le même fichier
#print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title']) # print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
print('easy ref:', sorted(Llink2, key=lambda f:str(f))) print('easy ref:', sorted(Llink2, key=lambda f: str(f)))
for f, _ in Llink2: for f, _ in Llink2:
Lunref.remove(f) Lunref.remove(f)
print('invalid:'+'\n'.join(str(f.api_fileid)+' '+str(f) for f in Linvalid)) print('invalid:' + '\n'.join(str(f.api_fileid) + ' ' + str(f) for f in Linvalid))
print('missing (', len(Lmissing), '):','\n'.join([str(e.api_id)+':'+repr(e)+'('+e.title+')' for e in sorted(Lmissing, key=lambda e:e.title)])) print('missing (', len(Lmissing), '):', '\n'.join(
print('unreferenced:'+'\n'.join(str(f) for f in sorted(Lunref, key=lambda e:e.title))) [str(e.api_id) + ':' + repr(e) + '(' + e.title + ')' for e in sorted(Lmissing, key=lambda e: e.title)]))
#print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref]))) print('unreferenced:' + '\n'.join(str(f) for f in sorted(Lunref, key=lambda e: e.title)))
# print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
return return
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle) # Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, film in enumerate(Linvalid): for i, film in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', film) print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', film)
try: try:
resp = api.delete_file(id=film.api_fileid) resp = api.delete_file(id=film.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not deleted') print('film ' + film.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Supprime les fichiers qui n'existent plus # Supprime les fichiers qui n'existent plus
for i, film in enumerate(Lmissing): for i, film in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', film) print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', film)
try: try:
resp = api.delete_file(id=film.api_fileid) resp = api.delete_file(id=film.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not deleted') print('film ' + film.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
@ -194,13 +207,13 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
i = 0 i = 0
for filmLoc, filmApi in Lrelink: for filmLoc, filmApi in Lrelink:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Lrelink))+']'+'relink:', filmApi.title) print('[' + str(i) + '/' + str(len(Lrelink)) + ']' + 'relink:', filmApi.title)
try: try:
api.put_file(id=filmApi.api_fileid, path=domain['server']+filmLoc.path, name=filmLoc.name) api.put_file(id=filmApi.api_fileid, path=domain['server'] + filmLoc.path, name=filmLoc.name)
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+filmApi.title+' not edited') print('film ' + filmApi.title + ' not edited')
raise Exception('end') raise Exception('end')
# Poste les ajouts de doubles # Poste les ajouts de doubles
@ -208,30 +221,31 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
for film, filmAPI in Llink: for film, filmAPI in Llink:
filmID = filmAPI.api_id filmID = filmAPI.api_id
i += 1 i += 1
print('['+str(i)+'/'+str(len(Llink))+']'+'link:', film.title) print('[' + str(i) + '/' + str(len(Llink)) + ']' + 'link:', film.title)
try: try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID)
if 'id' in resp: if 'id' in resp:
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not added') print('film ' + film.title + ' not added')
raise Exception('end') raise Exception('end')
# Poste les ajouts de doubles plus complexes # Poste les ajouts de doubles plus complexes
i = 0 i = 0
for film, filmID in Llink2: for film, filmID in Llink2:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Llink2))+']'+'link2:', film.title) print('[' + str(i) + '/' + str(len(Llink2)) + ']' + 'link2:', film.title)
try: try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID, **film.additional_info()) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID,
**film.additional_info())
if 'id' in resp: if 'id' in resp:
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not added') print('film ' + film.title + ' not added')
raise Exception('end') raise Exception('end')
# Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher) # Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher)
@ -239,7 +253,7 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
Lcannot_post = [] Lcannot_post = []
for film in Lunref: for film in Lunref:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Lunref))+']'+'post:', film.title, str(film.info.get('YEAR'))) print('[' + str(i) + '/' + str(len(Lunref)) + ']' + 'post:', film.title, str(film.info.get('YEAR')))
try: try:
posted = False posted = False
if 'YEAR' in film.info: # tente avec l'année spécifié en premier if 'YEAR' in film.info: # tente avec l'année spécifié en premier
@ -252,8 +266,8 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
posted = True posted = True
if posted: if posted:
print('post: path=',domain['server']+film.path) print('post: path=', domain['server'] + film.path)
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=resp["id"]) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=resp["id"])
if 'id' in resp: # id du file if 'id' in resp: # id du file
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
else: else:
@ -263,11 +277,12 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not posted') print('film ' + film.title + ' not posted')
raise Exception('end') raise Exception('end')
# TODO: traiter les films non postés (Lcannot_post) # TODO: traiter les films non postés (Lcannot_post)
def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok): def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
""" """
Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api
@ -277,7 +292,9 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
Lapi = Lepisode_api Lapi = Lepisode_api
# fichiers invalides # fichiers invalides
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules)) Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api # Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
@ -292,8 +309,8 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# récupère les séries, et les correspondances des noms locaux # récupère les séries, et les correspondances des noms locaux
APIseries = api.get_series(episodes=1) APIseries = api.get_series(episodes=1)
APIepisodes = api.get_episodes(files=1) APIepisodes = api.get_episodes(files=1)
APIepisodes_byid = {e['id']:e for e in APIepisodes} APIepisodes_byid = {e['id']: e for e in APIepisodes}
APIfiles_byid = {f.api_id:f for f in Lapi} APIfiles_byid = {f.api_id: f for f in Lapi}
series_id_bytitle = {} series_id_bytitle = {}
series_id_bysimpletitle = {} series_id_bysimpletitle = {}
for s in APIseries: for s in APIseries:
@ -314,30 +331,30 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle) # Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, episode in enumerate(Linvalid): for i, episode in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', episode) print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', episode)
try: try:
resp = api.delete_file(id=episode.api_fileid) resp = api.delete_file(id=episode.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not deleted') print('episode ' + episode.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Supprime les fichiers qui n'existent plus # Supprime les fichiers qui n'existent plus
for i, episode in enumerate(Lmissing): for i, episode in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', episode) print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', episode)
try: try:
resp = api.delete_file(id=episode.api_fileid) resp = api.delete_file(id=episode.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not deleted') print('episode ' + episode.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Poste les séries non présentes, récupère les références correspondantes # Poste les séries non présentes, récupère les références correspondantes
series_not_found = [] series_not_found = []
for i, serie in enumerate(Lunref_serie): for i, serie in enumerate(Lunref_serie):
print('['+str(i+1)+'/'+str(len(Lunref_serie))+']'+'unref_serie:', serie) print('[' + str(i + 1) + '/' + str(len(Lunref_serie)) + ']' + 'unref_serie:', serie)
try: try:
resp = api.post_serie(title=serie) resp = api.post_serie(title=serie)
if 'id' in resp: if 'id' in resp:
@ -348,7 +365,7 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
print('not found: ', resp) print('not found: ', resp)
except Exception as e: except Exception as e:
print(e) print(e)
print('serie '+serie+' not posted') print('serie ' + serie + ' not posted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
@ -357,25 +374,27 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Poste les episodes locaux # Poste les episodes locaux
for i, episode in enumerate(Lunref): for i, episode in enumerate(Lunref):
print('['+str(i+1)+'/'+str(len(Lunref))+']'+'post:', episode) print('[' + str(i + 1) + '/' + str(len(Lunref)) + ']' + 'post:', episode)
try: try:
serie_id = series_id_bytitle[episode.info['SERIE']] serie_id = series_id_bytitle[episode.info['SERIE']]
episode_num = int(episode.info['EPISODE_NUM']) episode_num = int(episode.info['EPISODE_NUM'])
season_num = int(episode.info['SEASON_NUM']) season_num = int(episode.info['SEASON_NUM'])
resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num) resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num)
if 'id' in resp: if 'id' in resp:
resp = api.post_file(path=domain['server']+episode.path, name=episode.name, type='episode', type_id=resp["id"]) resp = api.post_file(path=domain['server'] + episode.path, name=episode.name, type='episode',
print('response: ',resp) type_id=resp["id"])
print('response: ', resp)
else: else:
print('episode not posted:', resp) print('episode not posted:', resp)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not posted') print('episode ' + episode.title + ' not posted')
#raise Exception('end') # raise Exception('end')
time.sleep(1) time.sleep(1)
# TODO: traiter les séries non référencées (series_not_found) # TODO: traiter les séries non référencées (series_not_found)
def filter_by(L, f_prop): def filter_by(L, f_prop):
""" """
Sépare la liste L en deux listes, Sépare la liste L en deux listes,
@ -389,6 +408,7 @@ def filter_by(L, f_prop):
prop_false.append(e) prop_false.append(e)
return prop_true, prop_false return prop_true, prop_false
def update_find_by_common(L1, L2, f_prop): def update_find_by_common(L1, L2, f_prop):
""" """
Effectue une recherche de propriétés commune entre les éléments Effectue une recherche de propriétés commune entre les éléments
@ -402,6 +422,7 @@ def update_find_by_common(L1, L2, f_prop):
L2 = [e for e in L2 if e not in found2] L2 = [e for e in L2 if e not in found2]
return L1, L2, found return L1, L2, found
def find_by_common(L1, L2, f_prop): def find_by_common(L1, L2, f_prop):
""" """
Associe les éléments de <L1> et <L2> à travers leur propriétés Associe les éléments de <L1> et <L2> à travers leur propriétés
@ -435,6 +456,7 @@ def post_markers(api, file_, fileid):
api.post_file_subtitle(fileid, value=sub) api.post_file_subtitle(fileid, value=sub)
time.sleep(1) time.sleep(1)
def main(): def main():
""" """
Fonction principale du programme, réalise toute les opérations Fonction principale du programme, réalise toute les opérations
@ -449,9 +471,10 @@ def main():
rules = api.get_paths() rules = api.get_paths()
for fold in folders[1:]: for fold in folders[1:]:
applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']] applicable = [filerule.FileRule(re.escape(fold['path']) + '\\/' + r['regex'], conf) for r in rules if
int(r['indexer_folder_id']) == fold['id']]
visit_folder(fold, api, applicable, tokens) visit_folder(fold, api, applicable, tokens)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

63
piexel.py
View file

@ -1,28 +1,34 @@
#coding:utf-8 # coding:utf-8
import requests import requests
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
class PiexelErrors(Exception): class PiexelErrors(Exception):
pass pass
class InvalidToken(PiexelErrors): class InvalidToken(PiexelErrors):
pass pass
class ParameterError(PiexelErrors): class ParameterError(PiexelErrors):
pass pass
class InvalidResponse(PiexelErrors): class InvalidResponse(PiexelErrors):
pass pass
class Piexel: class Piexel:
def __init__(self, domain, app='', token='', endpoint='/api/'): def __init__(self, domain, app='', token='', endpoint='/api/'):
#self.app = app # self.app = app
#self.token = token # self.token = token
self.auth = HTTPBasicAuth(app, token) self.auth = HTTPBasicAuth(app, token)
self.domain = domain self.domain = domain
self.endpoint = endpoint self.endpoint = endpoint
def _get_response(self, controller, fields, request_type='get'): def _get_response(self, controller, fields, request_type='get', data=None):
""" """
Build response Build response
:param controller: controller à utiliser :param controller: controller à utiliser
@ -32,8 +38,14 @@ class Piexel:
if request_type == 'get': if request_type == 'get':
response = requests.get(url, auth=self.auth, params=fields) response = requests.get(url, auth=self.auth, params=fields)
elif request_type == 'post': elif request_type == 'post':
if data is not None:
response = requests.post(url, auth=self.auth, params=fields, json=data)
else:
response = requests.post(url, auth=self.auth, params=fields) response = requests.post(url, auth=self.auth, params=fields)
elif request_type == 'put': elif request_type == 'put':
if data is not None:
response = requests.put(url, auth=self.auth, params=fields, json=data)
else:
response = requests.put(url, auth=self.auth, params=fields) response = requests.put(url, auth=self.auth, params=fields)
elif request_type == 'delete': elif request_type == 'delete':
response = requests.delete(url, auth=self.auth, data=fields) response = requests.delete(url, auth=self.auth, data=fields)
@ -68,7 +80,8 @@ class Piexel:
Récupère les films Récupère les films
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [], **params) fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [],
**params)
return self._get_response('films', fields) return self._get_response('films', fields)
def get_series(self, **params): def get_series(self, **params):
@ -84,7 +97,8 @@ class Piexel:
Récupère les épisodes Récupère les épisodes
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [], **params) fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [],
**params)
return self._get_response('episodes', fields) return self._get_response('episodes', fields)
def get_tokens(self, **params): def get_tokens(self, **params):
@ -157,7 +171,7 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('languages/'+str(file)+'/attach', fields, 'post') return self._get_response('languages/' + str(file) + '/attach', fields, 'post')
def post_file_subtitle(self, file, **params): def post_file_subtitle(self, file, **params):
""" """
@ -165,7 +179,7 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('subtitle-languages/'+str(file)+'/attach', fields, 'post') return self._get_response('subtitle-languages/' + str(file) + '/attach', fields, 'post')
def post_file_qualities(self, file, **params): def post_file_qualities(self, file, **params):
""" """
@ -173,14 +187,15 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('qualities/'+str(file)+'/attach', fields, 'post') return self._get_response('qualities/' + str(file) + '/attach', fields, 'post')
def post_subtitle(self, **params): def post_subtitle(self, **params):
""" """
Ajoute un sous-titre Ajoute un sous-titre
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'], ['path', 'name', 'type', 'type_id'], **params) fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'],
['path', 'name', 'type', 'type_id'], **params)
return self._get_response('subtitles', fields, 'post') return self._get_response('subtitles', fields, 'post')
def post_serie(self, **params): def post_serie(self, **params):
@ -204,8 +219,9 @@ class Piexel:
Edite un fichier Edite un fichier
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [], **params) fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [],
return self._get_response('file/'+str(id), fields, 'put') **params)
return self._get_response('file/' + str(id), fields, 'put')
def delete_file(self, id, **params): def delete_file(self, id, **params):
""" """
@ -213,5 +229,26 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request([], [], **params) fields = self._get_request([], [], **params)
return self._get_response('file/'+str(id), fields, 'delete') return self._get_response('file/' + str(id), fields, 'delete')
def get_index(self, **params):
"""
Récupère les fichiers indexés
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'path', 'name', 'limit', 'first', 'is_referenced', 'like'], [], **params)
return self._get_response('index', fields)
def post_index(self, params):
"""
Ajoute un fichier
:param params: paramètres à passer
"""
return self._get_response('index', {}, 'post', params)
def delete_index(self, id):
"""
Supprime un fichier indéxé
:param id: id du fichier à supprimer
"""
return self._get_response('index/' + str(id), {}, 'delete')

16
tokenizer.py
View file

@ -1,7 +1,8 @@
#coding:utf-8 # coding:utf-8
import piexel import piexel
import re import re
class Tokenizer: class Tokenizer:
def __init__(self, conf, api): def __init__(self, conf, api):
self.conf = conf self.conf = conf
@ -28,13 +29,13 @@ class Tokenizer:
Les marqueurs sont enlevés, et rangés dans des listes. Les marqueurs sont enlevés, et rangés dans des listes.
retourne le nom privé des marqueurs, ainsi que le dictionnaire des marqueurs retourne le nom privé des marqueurs, ainsi que le dictionnaire des marqueurs
""" """
found = {'lang':[], 'quality':[], 'subtitle':[]} found = {'lang': [], 'quality': [], 'subtitle': []}
for step in self.steps: for step in self.steps:
for tok in self.get_tokens_step(step): for tok in self.get_tokens_step(step):
if(not bool(int(tok['case_sensitive']))): if (not bool(int(tok['case_sensitive']))):
reg = re.compile(r' '+tok['token']+r' ', re.IGNORECASE) reg = re.compile(r' ' + tok['token'] + r' ', re.IGNORECASE)
else: else:
reg = re.compile(r' '+tok['token']+r' ') reg = re.compile(r' ' + tok['token'] + r' ')
if reg.search(filename): if reg.search(filename):
for tok_lang in tok['languages']: for tok_lang in tok['languages']:
found['lang'].append(tok_lang['value']) found['lang'].append(tok_lang['value'])
@ -46,8 +47,3 @@ class Tokenizer:
for typ in found: for typ in found:
found[typ] = [e for e in found[typ] if e != 'N/A'] found[typ] = [e for e in found[typ] if e != 'N/A']
return filename, found return filename, found