piexel/piexel-indexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add indexed files

Browse source
This commit is contained in:
bdvllrs 2018-02-06 14:31:01 +01:00
parent cbbb8148b5
commit bf1fa8e3dd
7 changed files with 250 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -3,3 +3,4 @@ __pycache__/*
*.sublime-project *.sublime-project
*.sublime-workspace *.sublime-workspace
config.conf config.conf
.idea/*

18
config.py
View file

@ -1,4 +1,4 @@
#coding:utf-8 # coding:utf-8
from configobj import ConfigObj from configobj import ConfigObj
import shutil import shutil
import posixpath import posixpath
@ -12,8 +12,9 @@ class Config:
""" """
Gère le fichier de configuration Gère le fichier de configuration
""" """
def __init__(self): def __init__(self):
try: try:
self.obj = ConfigObj('config.conf', file_error=True) self.obj = ConfigObj('config.conf', file_error=True)
except: except:
@ -21,18 +22,18 @@ class Config:
print('creating config file') print('creating config file')
shutil.copyfile('default.conf', 'config.conf') shutil.copyfile('default.conf', 'config.conf')
self.obj = ConfigObj('config.conf') self.obj = ConfigObj('config.conf')
self.check_config() self.check_config()
print('config loaded') print('config loaded')
@property @property
def app(self): def app(self):
return self.obj['api']['app'] return self.obj['api']['app']
@property @property
def token(self): def token(self):
return self.obj['api']['token'] return self.obj['api']['token']
@property @property
def server(self): def server(self):
return self.obj['api']['server'] return self.obj['api']['server']
@ -40,15 +41,14 @@ class Config:
@property @property
def extensions(self): def extensions(self):
return self.obj['api']['extensions'] return self.obj['api']['extensions']
def check_config(self): def check_config(self):
if not 'api' in self.obj: if not 'api' in self.obj:
raise ConfigError("no 'api' section in config file") raise ConfigError("no 'api' section in config file")
for k in ['app', 'token', 'extensions', 'server']: for k in ['app', 'token', 'extensions', 'server']:
if not k in self.obj['api']: if not k in self.obj['api']:
raise ConfigError("no '"+k+"' value in config file, subsection 'api'") raise ConfigError("no '" + k + "' value in config file, subsection 'api'")
def is_valid_file(self, name): def is_valid_file(self, name):
_, ext = posixpath.splitext(name) _, ext = posixpath.splitext(name)
return ext[1:] in self.obj['api']['extensions'] return ext[1:] in self.obj['api']['extensions']

21
file.py
View file

@ -1,11 +1,13 @@
#coding:utf-8 # coding:utf-8
import posixpath import posixpath
import re import re
class File: class File:
""" """
Décrit une référence de fichier dans le disque Décrit une référence de fichier dans le disque
""" """
def __init__(self, path, name, info, api_id=None, api_fileid=None, api_fileable_type=None): def __init__(self, path, name, info, api_id=None, api_fileid=None, api_fileable_type=None):
self.path = path self.path = path
self.name = name self.name = name
@ -13,7 +15,7 @@ class File:
self.markers = {} self.markers = {}
self.api_id = api_id self.api_id = api_id
self.api_fileid = api_fileid self.api_fileid = api_fileid
self.fileable_type = {None: None, 'App\\Film':'film', 'App\\Episode':'episode'}[api_fileable_type] self.fileable_type = {None: None, 'App\\Film': 'film', 'App\\Episode': 'episode'}[api_fileable_type]
def get_ext(self): def get_ext(self):
""" """
@ -46,7 +48,7 @@ class File:
fname = fname.replace('(', ' ( ') fname = fname.replace('(', ' ( ')
fname = fname.replace(')', ' ) ') fname = fname.replace(')', ' ) ')
# 2) marqueurs # 2) marqueurs
fname, info = tok.tokenize(' '+fname+' ') fname, info = tok.tokenize(' ' + fname + ' ')
self.markers = info self.markers = info
# 3) minuscule # 3) minuscule
fname = fname.lower() fname = fname.lower()
@ -57,10 +59,10 @@ class File:
self.info['YEAR'] = m.group(1) self.info['YEAR'] = m.group(1)
if 'YEAR' in self.info: if 'YEAR' in self.info:
year = self.info['YEAR'] year = self.info['YEAR']
fname = fname.replace('['+str(year)+']', '') fname = fname.replace('[' + str(year) + ']', '')
fname = fname.replace('[ '+str(year)+' ]', '') fname = fname.replace('[ ' + str(year) + ' ]', '')
fname = fname.replace('['+str(year)+' ]', '') fname = fname.replace('[' + str(year) + ' ]', '')
fname = fname.replace('[ '+str(year)+']', '') fname = fname.replace('[ ' + str(year) + ']', '')
# 5) espaces en bout, centraux et rajoutés # 5) espaces en bout, centraux et rajoutés
fname = fname.lstrip().rstrip() fname = fname.lstrip().rstrip()
while ' ' in fname: while ' ' in fname:
@ -92,9 +94,8 @@ class File:
def __str__(self): def __str__(self):
if 'YEAR' in self.info: if 'YEAR' in self.info:
return str(self.path+'/'+self.name+' year:['+str(self.info['YEAR'])+']') return str(self.path + '/' + self.name + ' year:[' + str(self.info['YEAR']) + ']')
return str(self.path+'/'+self.name) return str(self.path + '/' + self.name)
def __repr__(self): def __repr__(self):
return str(self) return str(self)

60
filerule.py
View file

@ -2,37 +2,39 @@ import re
PATTERN_BASIC = re.compile(r'#(.+?)#') PATTERN_BASIC = re.compile(r'#(.+?)#')
def match_rules(filename, rules): def match_rules(filename, rules):
for r in rules: for r in rules:
match = r.apply(filename) match = r.apply(filename)
if match: if match:
return r, match return r, match
return None return None
class FileRule: class FileRule:
def __init__(self, rule, conf): def __init__(self, rule, conf):
# trouve les marqueurs dans la règle # trouve les marqueurs dans la règle
self.patterns = PATTERN_BASIC.findall(rule) self.patterns = PATTERN_BASIC.findall(rule)
rg = PATTERN_BASIC.sub(lambda m:self.regFor(match=m, conf=conf), rule) rg = PATTERN_BASIC.sub(lambda m: self.regFor(match=m, conf=conf), rule)
self.rule = re.compile(rg) # on garde la version compilée (donc optimisée) self.rule = re.compile(rg) # on garde la version compilée (donc optimisée)
@staticmethod @staticmethod
def regFor(match, conf): def regFor(match, conf):
name = match.group(0) name = match.group(0)
trunc_name = name[1:-1] trunc_name = name[1:-1]
if name == '#EXT#': if name == '#EXT#':
return '('+'|'.join(conf.extensions)+')' return '(' + '|'.join(conf.extensions) + ')'
elif name in ['#EPISODE_NUM#', '#SEASON_NUM#']: elif name in ['#EPISODE_NUM#', '#SEASON_NUM#']:
return '([0-9]+)' return '([0-9]+)'
return '([^/]+)' return '([^/]+)'
def apply(self, filename): def apply(self, filename):
# applique la règle à un objet file # applique la règle à un objet file
match = self.rule.match(filename) match = self.rule.match(filename)
if not match: if not match:
return None return None
# trouve les différents marqueurs de règle # trouve les différents marqueurs de règle
patterns = {} patterns = {}
for i in range(len(self.patterns)): for i in range(len(self.patterns)):
patterns[self.patterns[i]] = match.group(i+1) patterns[self.patterns[i]] = match.group(i + 1)
return patterns return patterns

183
main.py
View file

@ -1,4 +1,4 @@
#coding:utf-8 # coding:utf-8
''' '''
Pierre Cadart Pierre Cadart
@ -36,12 +36,13 @@ def ftpwalk(directory, ftp):
continue continue
if prop['type'] == 'dir': if prop['type'] == 'dir':
Ldirs.append(name) Ldirs.append(name)
to_visit.append(current+'/'+name) to_visit.append(current + '/' + name)
elif prop['type'] == 'file': elif prop['type'] == 'file':
Lfiles.append(name) Lfiles.append(name)
# ne construit pas la liste complète, # ne construit pas la liste complète,
# mais retourne les résultats intermédiaires # mais retourne les résultats intermédiaires
yield (current, Ldirs, Lfiles) yield (current, Ldirs, Lfiles)
def visit_folder(domain, api, rules, tok): def visit_folder(domain, api, rules, tok):
""" """
@ -55,39 +56,47 @@ def visit_folder(domain, api, rules, tok):
ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password']) ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password'])
ftp.encoding = 'UTF-8' ftp.encoding = 'UTF-8'
# Initialisation des listes de mises à jour # Initialisation des listes de mises à jour
L_missing = [] # fichiers non trouvés sur le serveur FTP L_missing = [] # fichiers non trouvés sur le serveur FTP
L_unreferenced = [] # fichiers non référencés dans l'API L_unreferenced = [] # fichiers non référencés dans l'API
L_moved = [] # fichiers déplacés sur le serveur FTP L_moved = [] # fichiers déplacés sur le serveur FTP
# Lecture des fichiers sur le serveur FTP # Lecture des fichiers sur le serveur FTP
Lloc = [] Lloc = []
indexed_files = []
for path, _, files in ftpwalk(domain['path'], ftp): for path, _, files in ftpwalk(domain['path'], ftp):
# Ajoute les fichiers correspondants aux extensions # Ajoute les fichiers correspondants aux extensions
for f in files: for f in files:
match = filerule.match_rules(path+'/'+f, rules) indexed_files.append({
"path": domain['server'] + path,
"name": f
})
match = filerule.match_rules(path + '/' + f, rules)
if match: if match:
#print('got match:',match[1], 'name:',path+'/'+f) # print('got match:',match[1], 'name:',path+'/'+f)
F = file.File(path, f, match[1]) F = file.File(path, f, match[1])
F.extract_title(tok) F.extract_title(tok)
Lloc.append(F) Lloc.append(F)
print('indexing all files :', len(indexed_files), ' files')
api.post_index(indexed_files)
ftp.close() ftp.close()
print('total loc for ',domain['server']+domain['path'],':', len(Lloc)) print('total loc for ', domain['server'] + domain['path'], ':', len(Lloc))
# Récupère les fichiers de l'api # Récupère les fichiers de l'api
Lapi = [] Lapi = []
for info in api.get_files(path=domain['server']+domain['path'], like=1, filable=1): for info in api.get_files(path=domain['server'] + domain['path'], like=1, filable=1):
nfo = {} nfo = {}
if ('filable' not in info) or ('filable_type' not in info): if ('filable' not in info) or ('filable_type' not in info):
print('nfo:', info) # le fileable associé a été supprimé print('nfo:', info) # le fileable associé a été supprimé
else: else:
year = int(info['filable']['release_date'][:4]) year = int(info['filable']['release_date'][:4])
nfo['YEAR'] = year nfo['YEAR'] = year
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'], api_fileid=info['id'], api_fileable_type=info['filable_type']) F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'],
api_fileid=info['id'], api_fileable_type=info['filable_type'])
F.extract_title(tok) F.extract_title(tok)
match = filerule.match_rules(F.path+'/'+F.name, rules) match = filerule.match_rules(F.path + '/' + F.name, rules)
if match: if match:
F.info = match[1] F.info = match[1]
F.info['YEAR'] = nfo['YEAR'] # may not work :-( F.info['YEAR'] = nfo['YEAR'] # may not work :-(
Lapi.append(F) Lapi.append(F)
print('total api for ',domain['server']+domain['path'],':', len(Lapi)) print('total api for ', domain['server'] + domain['path'], ':', len(Lapi))
# traite les films # traite les films
Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film'] Lfilm_loc = [f for f in Lloc if f.fileable_type == 'film']
@ -99,7 +108,8 @@ def visit_folder(domain, api, rules, tok):
Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode'] Lepisode_api = [f for f in Lapi if f.fileable_type == 'episode']
handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok) handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok)
print('visit finished ',domain['server']+domain['path']) print('visit finished ', domain['server'] + domain['path'])
def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok): def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
""" """
@ -109,84 +119,87 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
Lloc = Lfilm_loc Lloc = Lfilm_loc
Lapi = Lfilm_api Lapi = Lfilm_api
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc]))) # print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('loc titles:', '|'.join(sorted([f.title for f in Lloc]))) # print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
#print('\n'*2) # print('\n'*2)
#print('api titles:', '|'.join(sorted([f.title for f in Lapi]))) # print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
# supprime les dossiers de l'api (ils ne devraient pas apparaître) # supprime les dossiers de l'api (ils ne devraient pas apparaître)
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules)) Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api # Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
# Fichiers déplacés (ou copiés) localement # Fichiers déplacés (ou copiés) localement
Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f:f.name) Lunref, _, Lrelink = update_find_by_common(Lunref, Lapi, lambda f: f.name)
for floc, fapi in Lrelink: for floc, fapi in Lrelink:
if fApi in Lmissing: if fApi in Lmissing:
Lmissing.remove(fApi) Lmissing.remove(fApi)
print('moved/copied:', Lrelink) print('moved/copied:', Lrelink)
# Linke les fichiers du même titre (simplifié) au même film # Linke les fichiers du même titre (simplifié) au même film
Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f:f.title) Lunref, _, Llink = update_find_by_common(Lunref, Lapi, lambda f: f.title)
for f, fApi in Llink: for f, fApi in Llink:
if fApi in Lmissing: if fApi in Lmissing:
Lmissing.remove(fApi) Lmissing.remove(fApi)
print('doubles:', sorted(Llink, key=lambda f:str(f[0]))) print('doubles:', sorted(Llink, key=lambda f: str(f[0])))
# Linke les films par nom si possible # Linke les films par nom si possible
APIfilms = api.get_films() APIfilms = api.get_films()
API_alltitles = [] API_alltitles = []
for f in APIfilms: for f in APIfilms:
if f['title']: if f['title']:
t = f['title'].replace(' ','').lower() t = f['title'].replace(' ', '').lower()
if len(t) > 2: if len(t) > 2:
if t not in [e[0] for e in API_alltitles]: if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id'])) API_alltitles.append((t, f['id']))
if f['title_vo']: if f['title_vo']:
t = f['title_vo'].replace(' ','').lower() t = f['title_vo'].replace(' ', '').lower()
if len(t) > 2: if len(t) > 2:
if t not in [e[0] for e in API_alltitles]: if t not in [e[0] for e in API_alltitles]:
API_alltitles.append((t, f['id'])) API_alltitles.append((t, f['id']))
Llink2 = [] Llink2 = []
for film in Lunref: for film in Lunref:
for title, fid in API_alltitles: for title, fid in API_alltitles:
if title==film.title: if title == film.title:
Llink2.append((film, fid)) Llink2.append((film, fid))
break # pour ne pas référencer deux fois le même fichier break # pour ne pas référencer deux fois le même fichier
#print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title']) # print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
print('easy ref:', sorted(Llink2, key=lambda f:str(f))) print('easy ref:', sorted(Llink2, key=lambda f: str(f)))
for f, _ in Llink2: for f, _ in Llink2:
Lunref.remove(f) Lunref.remove(f)
print('invalid:'+'\n'.join(str(f.api_fileid)+' '+str(f) for f in Linvalid)) print('invalid:' + '\n'.join(str(f.api_fileid) + ' ' + str(f) for f in Linvalid))
print('missing (', len(Lmissing), '):','\n'.join([str(e.api_id)+':'+repr(e)+'('+e.title+')' for e in sorted(Lmissing, key=lambda e:e.title)])) print('missing (', len(Lmissing), '):', '\n'.join(
print('unreferenced:'+'\n'.join(str(f) for f in sorted(Lunref, key=lambda e:e.title))) [str(e.api_id) + ':' + repr(e) + '(' + e.title + ')' for e in sorted(Lmissing, key=lambda e: e.title)]))
#print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref]))) print('unreferenced:' + '\n'.join(str(f) for f in sorted(Lunref, key=lambda e: e.title)))
# print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
return return
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle) # Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, film in enumerate(Linvalid): for i, film in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', film) print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', film)
try: try:
resp = api.delete_file(id=film.api_fileid) resp = api.delete_file(id=film.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not deleted') print('film ' + film.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Supprime les fichiers qui n'existent plus # Supprime les fichiers qui n'existent plus
for i, film in enumerate(Lmissing): for i, film in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', film) print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', film)
try: try:
resp = api.delete_file(id=film.api_fileid) resp = api.delete_file(id=film.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not deleted') print('film ' + film.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
@ -194,67 +207,68 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
i = 0 i = 0
for filmLoc, filmApi in Lrelink: for filmLoc, filmApi in Lrelink:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Lrelink))+']'+'relink:', filmApi.title) print('[' + str(i) + '/' + str(len(Lrelink)) + ']' + 'relink:', filmApi.title)
try: try:
api.put_file(id=filmApi.api_fileid, path=domain['server']+filmLoc.path, name=filmLoc.name) api.put_file(id=filmApi.api_fileid, path=domain['server'] + filmLoc.path, name=filmLoc.name)
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+filmApi.title+' not edited') print('film ' + filmApi.title + ' not edited')
raise Exception('end') raise Exception('end')
# Poste les ajouts de doubles # Poste les ajouts de doubles
i = 0 i = 0
for film, filmAPI in Llink: for film, filmAPI in Llink:
filmID = filmAPI.api_id filmID = filmAPI.api_id
i += 1 i += 1
print('['+str(i)+'/'+str(len(Llink))+']'+'link:', film.title) print('[' + str(i) + '/' + str(len(Llink)) + ']' + 'link:', film.title)
try: try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID)
if 'id' in resp: if 'id' in resp:
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not added') print('film ' + film.title + ' not added')
raise Exception('end') raise Exception('end')
# Poste les ajouts de doubles plus complexes # Poste les ajouts de doubles plus complexes
i = 0 i = 0
for film, filmID in Llink2: for film, filmID in Llink2:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Llink2))+']'+'link2:', film.title) print('[' + str(i) + '/' + str(len(Llink2)) + ']' + 'link2:', film.title)
try: try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID, **film.additional_info()) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID,
**film.additional_info())
if 'id' in resp: if 'id' in resp:
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not added') print('film ' + film.title + ' not added')
raise Exception('end') raise Exception('end')
# Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher) # Poste tout les films locaux (doit faire une reqête Tmdb, qui peut ne pas marcher)
i = 0 i = 0
Lcannot_post = [] Lcannot_post = []
for film in Lunref: for film in Lunref:
i += 1 i += 1
print('['+str(i)+'/'+str(len(Lunref))+']'+'post:', film.title, str(film.info.get('YEAR'))) print('[' + str(i) + '/' + str(len(Lunref)) + ']' + 'post:', film.title, str(film.info.get('YEAR')))
try: try:
posted = False posted = False
if 'YEAR' in film.info: # tente avec l'année spécifié en premier if 'YEAR' in film.info: # tente avec l'année spécifié en premier
resp = api.post_film(title=film.title, year=film.info['YEAR']) resp = api.post_film(title=film.title, year=film.info['YEAR'])
if "id" in resp: # id du film if "id" in resp: # id du film
posted = True posted = True
if not posted: if not posted:
resp = api.post_film(title=film.title) resp = api.post_film(title=film.title)
if "id" in resp: # id du film if "id" in resp: # id du film
posted = True posted = True
if posted: if posted:
print('post: path=',domain['server']+film.path) print('post: path=', domain['server'] + film.path)
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=resp["id"]) resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=resp["id"])
if 'id' in resp: # id du file if 'id' in resp: # id du file
post_markers(api, film, resp['id']) post_markers(api, film, resp['id'])
else: else:
Lcannot_post.append(film) Lcannot_post.append(film)
@ -263,11 +277,12 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
print('film '+film.title+' not posted') print('film ' + film.title + ' not posted')
raise Exception('end') raise Exception('end')
# TODO: traiter les films non postés (Lcannot_post) # TODO: traiter les films non postés (Lcannot_post)
def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok): def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
""" """
Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api
@ -277,11 +292,13 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
Lapi = Lepisode_api Lapi = Lepisode_api
# fichiers invalides # fichiers invalides
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules)) Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api # Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés Lunref = [f for f in Lloc if f not in Lapi] # fichiers non référencés
# de même avec les noms de séries # de même avec les noms de séries
Lseries_loc = list(set([f.info['SERIE'] for f in Lloc])) Lseries_loc = list(set([f.info['SERIE'] for f in Lloc]))
@ -292,8 +309,8 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# récupère les séries, et les correspondances des noms locaux # récupère les séries, et les correspondances des noms locaux
APIseries = api.get_series(episodes=1) APIseries = api.get_series(episodes=1)
APIepisodes = api.get_episodes(files=1) APIepisodes = api.get_episodes(files=1)
APIepisodes_byid = {e['id']:e for e in APIepisodes} APIepisodes_byid = {e['id']: e for e in APIepisodes}
APIfiles_byid = {f.api_id:f for f in Lapi} APIfiles_byid = {f.api_id: f for f in Lapi}
series_id_bytitle = {} series_id_bytitle = {}
series_id_bysimpletitle = {} series_id_bysimpletitle = {}
for s in APIseries: for s in APIseries:
@ -314,30 +331,30 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle) # Supprime les fichiers invalides (dossiers/ ne répondent à aucune règle)
for i, episode in enumerate(Linvalid): for i, episode in enumerate(Linvalid):
print('['+str(i+1)+'/'+str(len(Linvalid))+']'+'invalid:', episode) print('[' + str(i + 1) + '/' + str(len(Linvalid)) + ']' + 'invalid:', episode)
try: try:
resp = api.delete_file(id=episode.api_fileid) resp = api.delete_file(id=episode.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not deleted') print('episode ' + episode.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Supprime les fichiers qui n'existent plus # Supprime les fichiers qui n'existent plus
for i, episode in enumerate(Lmissing): for i, episode in enumerate(Lmissing):
print('['+str(i+1)+'/'+str(len(Lmissing))+']'+'missing:', episode) print('[' + str(i + 1) + '/' + str(len(Lmissing)) + ']' + 'missing:', episode)
try: try:
resp = api.delete_file(id=episode.api_fileid) resp = api.delete_file(id=episode.api_fileid)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not deleted') print('episode ' + episode.title + ' not deleted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
# Poste les séries non présentes, récupère les références correspondantes # Poste les séries non présentes, récupère les références correspondantes
series_not_found = [] series_not_found = []
for i, serie in enumerate(Lunref_serie): for i, serie in enumerate(Lunref_serie):
print('['+str(i+1)+'/'+str(len(Lunref_serie))+']'+'unref_serie:', serie) print('[' + str(i + 1) + '/' + str(len(Lunref_serie)) + ']' + 'unref_serie:', serie)
try: try:
resp = api.post_serie(title=serie) resp = api.post_serie(title=serie)
if 'id' in resp: if 'id' in resp:
@ -348,7 +365,7 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
print('not found: ', resp) print('not found: ', resp)
except Exception as e: except Exception as e:
print(e) print(e)
print('serie '+serie+' not posted') print('serie ' + serie + ' not posted')
raise Exception('end') raise Exception('end')
time.sleep(1) time.sleep(1)
@ -357,25 +374,27 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# Poste les episodes locaux # Poste les episodes locaux
for i, episode in enumerate(Lunref): for i, episode in enumerate(Lunref):
print('['+str(i+1)+'/'+str(len(Lunref))+']'+'post:', episode) print('[' + str(i + 1) + '/' + str(len(Lunref)) + ']' + 'post:', episode)
try: try:
serie_id = series_id_bytitle[episode.info['SERIE']] serie_id = series_id_bytitle[episode.info['SERIE']]
episode_num = int(episode.info['EPISODE_NUM']) episode_num = int(episode.info['EPISODE_NUM'])
season_num = int(episode.info['SEASON_NUM']) season_num = int(episode.info['SEASON_NUM'])
resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num) resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num)
if 'id' in resp: if 'id' in resp:
resp = api.post_file(path=domain['server']+episode.path, name=episode.name, type='episode', type_id=resp["id"]) resp = api.post_file(path=domain['server'] + episode.path, name=episode.name, type='episode',
print('response: ',resp) type_id=resp["id"])
print('response: ', resp)
else: else:
print('episode not posted:', resp) print('episode not posted:', resp)
except Exception as e: except Exception as e:
print(e) print(e)
print('episode '+episode.title+' not posted') print('episode ' + episode.title + ' not posted')
#raise Exception('end') # raise Exception('end')
time.sleep(1) time.sleep(1)
# TODO: traiter les séries non référencées (series_not_found) # TODO: traiter les séries non référencées (series_not_found)
def filter_by(L, f_prop): def filter_by(L, f_prop):
""" """
Sépare la liste L en deux listes, Sépare la liste L en deux listes,
@ -389,6 +408,7 @@ def filter_by(L, f_prop):
prop_false.append(e) prop_false.append(e)
return prop_true, prop_false return prop_true, prop_false
def update_find_by_common(L1, L2, f_prop): def update_find_by_common(L1, L2, f_prop):
""" """
Effectue une recherche de propriétés commune entre les éléments Effectue une recherche de propriétés commune entre les éléments
@ -402,6 +422,7 @@ def update_find_by_common(L1, L2, f_prop):
L2 = [e for e in L2 if e not in found2] L2 = [e for e in L2 if e not in found2]
return L1, L2, found return L1, L2, found
def find_by_common(L1, L2, f_prop): def find_by_common(L1, L2, f_prop):
""" """
Associe les éléments de <L1> et <L2> à travers leur propriétés Associe les éléments de <L1> et <L2> à travers leur propriétés
@ -435,6 +456,7 @@ def post_markers(api, file_, fileid):
api.post_file_subtitle(fileid, value=sub) api.post_file_subtitle(fileid, value=sub)
time.sleep(1) time.sleep(1)
def main(): def main():
""" """
Fonction principale du programme, réalise toute les opérations Fonction principale du programme, réalise toute les opérations
@ -449,9 +471,10 @@ def main():
rules = api.get_paths() rules = api.get_paths()
for fold in folders[1:]: for fold in folders[1:]:
applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']] applicable = [filerule.FileRule(re.escape(fold['path']) + '\\/' + r['regex'], conf) for r in rules if
int(r['indexer_folder_id']) == fold['id']]
visit_folder(fold, api, applicable, tokens) visit_folder(fold, api, applicable, tokens)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

69
piexel.py
View file

@ -1,28 +1,34 @@
#coding:utf-8 # coding:utf-8
import requests import requests
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
class PiexelErrors(Exception): class PiexelErrors(Exception):
pass pass
class InvalidToken(PiexelErrors): class InvalidToken(PiexelErrors):
pass pass
class ParameterError(PiexelErrors): class ParameterError(PiexelErrors):
pass pass
class InvalidResponse(PiexelErrors): class InvalidResponse(PiexelErrors):
pass pass
class Piexel: class Piexel:
def __init__(self, domain, app='', token='', endpoint='/api/'): def __init__(self, domain, app='', token='', endpoint='/api/'):
#self.app = app # self.app = app
#self.token = token # self.token = token
self.auth = HTTPBasicAuth(app, token) self.auth = HTTPBasicAuth(app, token)
self.domain = domain self.domain = domain
self.endpoint = endpoint self.endpoint = endpoint
def _get_response(self, controller, fields, request_type='get'): def _get_response(self, controller, fields, request_type='get', data=None):
""" """
Build response Build response
:param controller: controller à utiliser :param controller: controller à utiliser
@ -32,9 +38,15 @@ class Piexel:
if request_type == 'get': if request_type == 'get':
response = requests.get(url, auth=self.auth, params=fields) response = requests.get(url, auth=self.auth, params=fields)
elif request_type == 'post': elif request_type == 'post':
response = requests.post(url, auth=self.auth, params=fields) if data is not None:
response = requests.post(url, auth=self.auth, params=fields, json=data)
else:
response = requests.post(url, auth=self.auth, params=fields)
elif request_type == 'put': elif request_type == 'put':
response = requests.put(url, auth=self.auth, params=fields) if data is not None:
response = requests.put(url, auth=self.auth, params=fields, json=data)
else:
response = requests.put(url, auth=self.auth, params=fields)
elif request_type == 'delete': elif request_type == 'delete':
response = requests.delete(url, auth=self.auth, data=fields) response = requests.delete(url, auth=self.auth, data=fields)
response.encoding = 'utf-8' response.encoding = 'utf-8'
@ -67,8 +79,9 @@ class Piexel:
""" """
Récupère les films Récupère les films
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [], **params) fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [],
**params)
return self._get_response('films', fields) return self._get_response('films', fields)
def get_series(self, **params): def get_series(self, **params):
@ -84,7 +97,8 @@ class Piexel:
Récupère les épisodes Récupère les épisodes
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [], **params) fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [],
**params)
return self._get_response('episodes', fields) return self._get_response('episodes', fields)
def get_tokens(self, **params): def get_tokens(self, **params):
@ -157,7 +171,7 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('languages/'+str(file)+'/attach', fields, 'post') return self._get_response('languages/' + str(file) + '/attach', fields, 'post')
def post_file_subtitle(self, file, **params): def post_file_subtitle(self, file, **params):
""" """
@ -165,7 +179,7 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('subtitle-languages/'+str(file)+'/attach', fields, 'post') return self._get_response('subtitle-languages/' + str(file) + '/attach', fields, 'post')
def post_file_qualities(self, file, **params): def post_file_qualities(self, file, **params):
""" """
@ -173,14 +187,15 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['value'], ['value'], **params) fields = self._get_request(['value'], ['value'], **params)
return self._get_response('qualities/'+str(file)+'/attach', fields, 'post') return self._get_response('qualities/' + str(file) + '/attach', fields, 'post')
def post_subtitle(self, **params): def post_subtitle(self, **params):
""" """
Ajoute un sous-titre Ajoute un sous-titre
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'], ['path', 'name', 'type', 'type_id'], **params) fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'],
['path', 'name', 'type', 'type_id'], **params)
return self._get_response('subtitles', fields, 'post') return self._get_response('subtitles', fields, 'post')
def post_serie(self, **params): def post_serie(self, **params):
@ -204,8 +219,9 @@ class Piexel:
Edite un fichier Edite un fichier
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [], **params) fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [],
return self._get_response('file/'+str(id), fields, 'put') **params)
return self._get_response('file/' + str(id), fields, 'put')
def delete_file(self, id, **params): def delete_file(self, id, **params):
""" """
@ -213,5 +229,26 @@ class Piexel:
:param params: paramètres à passer :param params: paramètres à passer
""" """
fields = self._get_request([], [], **params) fields = self._get_request([], [], **params)
return self._get_response('file/'+str(id), fields, 'delete') return self._get_response('file/' + str(id), fields, 'delete')
def get_index(self, **params):
"""
Récupère les fichiers indexés
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'path', 'name', 'limit', 'first', 'is_referenced', 'like'], [], **params)
return self._get_response('index', fields)
def post_index(self, params):
"""
Ajoute un fichier
:param params: paramètres à passer
"""
return self._get_response('index', {}, 'post', params)
def delete_index(self, id):
"""
Supprime un fichier indéxé
:param id: id du fichier à supprimer
"""
return self._get_response('index/' + str(id), {}, 'delete')

88
tokenizer.py
View file

@ -1,53 +1,49 @@
#coding:utf-8 # coding:utf-8
import piexel import piexel
import re import re
class Tokenizer: class Tokenizer:
def __init__(self, conf, api): def __init__(self, conf, api):
self.conf = conf self.conf = conf
self.reload_tokens(api) self.reload_tokens(api)
def reload_tokens(self, api):
"""
Charge les tokens depuis l'API,
et initialise la liste des étapes
"""
self.tk = api.get_tokens()
self.steps = list(set(t['step'] for t in self.tk))
self.steps.sort()
def get_tokens_step(self, step):
"""
Retourne tout les tokens de l'étape <step>
"""
return [t for t in self.tk if t['step'] == step]
def tokenize(self, filename):
"""
Analyse <filename> pour trouver tous ses marqueurs.
Les marqueurs sont enlevés, et rangés dans des listes.
retourne le nom privé des marqueurs, ainsi que le dictionnaire des marqueurs
"""
found = {'lang':[], 'quality':[], 'subtitle':[]}
for step in self.steps:
for tok in self.get_tokens_step(step):
if(not bool(int(tok['case_sensitive']))):
reg = re.compile(r' '+tok['token']+r' ', re.IGNORECASE)
else:
reg = re.compile(r' '+tok['token']+r' ')
if reg.search(filename):
for tok_lang in tok['languages']:
found['lang'].append(tok_lang['value'])
for tok_qual in tok['qualities']:
found['quality'].append(tok_qual['value'])
for tok_sub in tok['subtitle_languages']:
found['subtitle'].append(tok_sub['value'])
filename = reg.sub(' ', filename)
for typ in found:
found[typ] = [e for e in found[typ] if e != 'N/A']
return filename, found
def reload_tokens(self, api):
"""
Charge les tokens depuis l'API,
et initialise la liste des étapes
"""
self.tk = api.get_tokens()
self.steps = list(set(t['step'] for t in self.tk))
self.steps.sort()
def get_tokens_step(self, step):
"""
Retourne tout les tokens de l'étape <step>
"""
return [t for t in self.tk if t['step'] == step]
def tokenize(self, filename):
"""
Analyse <filename> pour trouver tous ses marqueurs.
Les marqueurs sont enlevés, et rangés dans des listes.
retourne le nom privé des marqueurs, ainsi que le dictionnaire des marqueurs
"""
found = {'lang': [], 'quality': [], 'subtitle': []}
for step in self.steps:
for tok in self.get_tokens_step(step):
if (not bool(int(tok['case_sensitive']))):
reg = re.compile(r' ' + tok['token'] + r' ', re.IGNORECASE)
else:
reg = re.compile(r' ' + tok['token'] + r' ')
if reg.search(filename):
for tok_lang in tok['languages']:
found['lang'].append(tok_lang['value'])
for tok_qual in tok['qualities']:
found['quality'].append(tok_qual['value'])
for tok_sub in tok['subtitle_languages']:
found['subtitle'].append(tok_sub['value'])
filename = reg.sub(' ', filename)
for typ in found:
found[typ] = [e for e in found[typ] if e != 'N/A']
return filename, found