piexel/piexel-indexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tout modifié pour la nouvelle API

Browse source
This commit is contained in:
redstorm45 2017-10-29 20:58:32 +01:00
parent d1de6ffe09
commit cb4ea4d419
5 changed files with 86 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

52
file.py
View file

@ -1,21 +1,16 @@
#coding:utf-8 #coding:utf-8
import posixpath import posixpath
import re
class File: class File:
""" """
Décrit une référence de fichier dans le disque Décrit une référence de fichier dans le disque
""" """
def __init__(self, path, name,api_id=None): def __init__(self, path, name, info, api_id=None):
self.path = path self.path = path
self.name = name self.name = name
self.fullname = path+name self.info = info
''' self.markers = {}
self.info_lang = []
self.info_quality = []
self.info_subtitles = []
self.title = ''
self.simple_name = self.title.replace(' ', '')
'''
self.api_id = api_id self.api_id = api_id
def get_ext(self): def get_ext(self):
@ -23,37 +18,46 @@ class File:
return ext return ext
def without_ext(self): def without_ext(self):
_, ext = posixpath.splitext(self.name) n, ext = posixpath.splitext(self.name)
return ext return n
def _extract_title(self, title, tok): def extract_title(self, tok):
# extrait un titre de film depuis la variable #TITLE# # extrait un titre de film depuis la variable #TITLE#
fname, ext = posixpath.splitext(title) if 'TITLE' in self.info:
fname = self.info['TITLE']
else:
fname, ext = posixpath.splitext(self.name)
# 1) séparateurs # 1) séparateurs
fname = fname.replace('.', ' ') fname = fname.replace('.', ' ')
fname = fname.replace('_', ' ') fname = fname.replace('_', ' ')
fname = fname.replace('-', ' ') fname = fname.replace('-', ' ')
# 2) marqueurs # 2) marqueurs
fname, info = tok.tokenize(fname) fname, info = tok.tokenize(fname)
self.markers = info
# 3) minuscule # 3) minuscule
fname = fname.lower() fname = fname.lower()
# 4) année # 4) année
if self.year is not None: if not 'YEAR' in self.info:
fname = fname.replace('['+str(self.year)+']', '') m = re.match(r'.*\[ ?(\d{4}) ?\].*', fname)
fname = fname.replace('[ '+str(self.year)+' ]', '') if m:
fname = fname.replace('['+str(self.year)+' ]', '') self.info['YEAR'] = m.group(1)
fname = fname.replace('[ '+str(self.year)+']', '') if 'YEAR' in self.info:
year = self.info['YEAR']
fname = fname.replace('['+str(year)+']', '')
fname = fname.replace('[ '+str(year)+' ]', '')
fname = fname.replace('['+str(year)+' ]', '')
fname = fname.replace('[ '+str(year)+']', '')
# 5) espaces en bout, centraux et rajoutés # 5) espaces en bout, centraux et rajoutés
fname = fname.lstrip().rstrip() fname = fname.lstrip().rstrip()
while ' ' in fname: while ' ' in fname:
fname = fname.replace(' ', ' ') fname = fname.replace(' ', ' ')
return fname self.title = fname
def filename_same(self, other): def filename_same(self, other):
# Compare les noms de fichiers de self et de other # Compare les noms de fichiers de self et de other
# En supprimant les espaces, la date, et les marques de qualitée # En supprimant les espaces, la date, et les marques de qualitée
return self.simple_name == other.simple_name return self.name == other.name
'''
def additional_info(self): def additional_info(self):
""" """
Info supplémentaires sur la langue, la qualitée et les sous-titres du fichier Info supplémentaires sur la langue, la qualitée et les sous-titres du fichier
@ -87,7 +91,7 @@ class File:
if m in self.info_subtitles: if m in self.info_subtitles:
info['subtitles'] = 'EN' info['subtitles'] = 'EN'
return info return info
'''
def __eq__(self, other): def __eq__(self, other):
return (self.path, self.name) == (other.path, other.name) return (self.path, self.name) == (other.path, other.name)
@ -95,7 +99,9 @@ class File:
return hash((self.path, self.name)) return hash((self.path, self.name))
def __str__(self): def __str__(self):
return str(self.path+'/'+self.name+' year:['+str(self.year)+']') if 'YEAR' in self.info:
return str(self.path+'/'+self.name+' year:['+str(self.info['YEAR'])+']')
return str(self.path+'/'+self.name)
def __repr__(self): def __repr__(self):
return str(self) return str(self)

12
filerule.py
View file

@ -1,5 +1,6 @@
import re import re
PATTERN_BASIC = re.compile(r'#(.+?)#')
def match_rules(filename, rules): def match_rules(filename, rules):
for r in rules: for r in rules:
@ -11,17 +12,18 @@ def match_rules(filename, rules):
class FileRule: class FileRule:
def __init__(self, rule, conf): def __init__(self, rule, conf):
# trouve les marqueurs dans la règle # trouve les marqueurs dans la règle
reg = re.compile(r'#(.+?)#') self.patterns = PATTERN_BASIC.findall(rule)
self.patterns = reg.findall(rule) rg = PATTERN_BASIC.sub(lambda m:self.regFor(match=m, conf=conf), rule)
rg = reg.sub(lambda m:self.regFor(match=m, conf=conf), rule) self.rule = re.compile(rg) # on garde la version compilée (donc optimisée)
self.rule = re.compile(rg)
print(self.patterns, self.rule)
@staticmethod @staticmethod
def regFor(match, conf): def regFor(match, conf):
name = match.group(0) name = match.group(0)
trunc_name = name[1:-1]
if name == '#EXT#': if name == '#EXT#':
return '('+'|'.join(conf.extensions)+')' return '('+'|'.join(conf.extensions)+')'
elif name in ['#EPISODE_NUM#', '#SEASON_NUM#']:
return '([0-9]+)'
return '([^/]+)' return '([^/]+)'
def apply(self, filename): def apply(self, filename):

41
main.py
View file

@ -42,7 +42,7 @@ def ftpwalk(directory, ftp):
# mais retourne les résultats intermédiaires # mais retourne les résultats intermédiaires
yield (current, Ldirs, Lfiles) yield (current, Ldirs, Lfiles)
def visit_folder(domain, api, rules): def visit_folder(domain, api, rules, tok):
# Connection au serveur # Connection au serveur
print('connect to:', domain['server']) print('connect to:', domain['server'])
ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password']) ftp = FTP(domain['server'][6:], user=domain['username'], passwd=domain['password'])
@ -58,21 +58,33 @@ def visit_folder(domain, api, rules):
for f in files: for f in files:
match = filerule.match_rules(path+'/'+f, rules) match = filerule.match_rules(path+'/'+f, rules)
if match: if match:
print('got match:',match[1], 'name:',path+'/'+f) #print('got match:',match[1], 'name:',path+'/'+f)
F = file.File(path+'/'+f, match[1]) F = file.File(path, f, match[1])
F.extract_title(tok)
Lloc.append(F) Lloc.append(F)
ftp.close() ftp.close()
print('total:',len(Lloc)) print('total:',len(Lloc))
exit(0)
# Application des règles de chemins
# Récupère les fichiers de l'api # Récupère les fichiers de l'api
Lapi = [] Lapi = []
for info in api.get_files(path='ftp://'+domain['server']+domain['path'], like=1): for info in api.get_files(path=domain['server']+domain['path'], like=1, filable=1):
Lapi.append(file.File(info['path'][len('ftp://'+domain['server']):], info['name'], api_id=info['filable_id'])) nfo = {}
if not info['filable']:
print('nfo:', info)
else:
year = int(info['filable']['release_date'][:4])
nfo['YEAR'] = year
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'])
F.extract_title(tok)
Lapi.append(F)
print('got api for ',domain['server']+domain['path'],':', len(Lapi))
print('loc titles:', '|'.join(sorted([f.title for f in Lloc])))
print('\n'*2)
print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
return
# supprime les dossiers de l'api # supprime les dossiers de l'api
Lapi = [f for f in Lapi if conf.is_valid_file(f.name)] Lapi = [f for f in Lapi if tok.conf.is_valid_file(f.name)]
# Compare avec la liste de l'api # Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
@ -117,19 +129,20 @@ def visit_folder(domain, api, rules):
Llink2 = [] Llink2 = []
for film in Lunref: for film in Lunref:
for title, fid in API_alltitles: for title, fid in API_alltitles:
if title==film.simple_name: if title==film.title:
Llink2.append((film, fid)) Llink2.append((film, fid))
break break
#print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title']) #print(film, ' <-> ', [f for f in APIfilms if f['id']==fid][0]['title'])
print('easy ref:', sorted(Llink2, key=lambda f:str(f))) print('easy ref:', sorted(Llink2, key=lambda f:str(f)))
for f, _ in Llink2: for f, _ in Llink2:
Lunref.remove(f) Lunref.remove(f)
print('missing:',[str(e.api_id)+':'+repr(e) for e in Lmissing]) print('missing (', len(Lmissing), '):',[str(e.api_id)+':'+repr(e) for e in Lmissing])
print('\n'*3) print('\n'*3)
print('unreferenced:','\n'.join(str(f) for f in Lunref)) print('unreferenced:','|'.join(str(f) for f in Lunref))
print('\n'*3) print('\n'*3)
print('unreferenced titles:', '\n'.join(sorted([f.title for f in Lunref]))) print('unreferenced titles:', '|'.join(sorted([f.title for f in Lunref])))
''' '''
# Put les renommages / déplacements # Put les renommages / déplacements
i = 0 i = 0
@ -222,7 +235,7 @@ def main():
for fold in folders: for fold in folders:
applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']] applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']]
visit_folder(fold, api, applicable) visit_folder(fold, api, applicable, tokens)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

53
piexel.py
View file

@ -1,5 +1,6 @@
#coding:utf-8 #coding:utf-8
import requests import requests
from requests.auth import HTTPBasicAuth
class PiexelErrors(Exception): class PiexelErrors(Exception):
pass pass
@ -15,8 +16,9 @@ class InvalidResponse(PiexelErrors):
class Piexel: class Piexel:
def __init__(self, domain, app='', token='', endpoint='/api/'): def __init__(self, domain, app='', token='', endpoint='/api/'):
self.app = app #self.app = app
self.token = token #self.token = token
self.auth = HTTPBasicAuth(app, token)
self.domain = domain self.domain = domain
self.endpoint = endpoint self.endpoint = endpoint
@ -26,23 +28,24 @@ class Piexel:
:param controller: controller à utiliser :param controller: controller à utiliser
:param fields: champ à passer :param fields: champ à passer
""" """
fields['app'] = self.app #fields['app'] = self.app
fields['token'] = self.token #fields['token'] = self.token
#fields['auth'] = self.auth
url = self.domain + self.endpoint + controller url = self.domain + self.endpoint + controller
if request_type == 'get': if request_type == 'get':
response = requests.get(url, fields) response = requests.get(url, auth=self.auth, params=fields)
elif request_type == 'post': elif request_type == 'post':
response = requests.post(url, fields) response = requests.post(url, auth=self.auth, params=fields)
elif request_type == 'put': elif request_type == 'put':
response = requests.put(url, fields) response = requests.put(url, auth=self.auth, params=fields)
elif request_type == 'delete': elif request_type == 'delete':
response = requests.delete(url, data=fields) response = requests.delete(url, auth=self.auth, data=fields)
response.encoding = 'utf-8' response.encoding = 'utf-8'
code = response.status_code code = response.status_code
try: try:
data = response.json() data = response.json()
except: except:
raise InvalidResponse(response.text) raise InvalidResponse(url, fields, response.text)
if code == 403: # FORBIDDEN if code == 403: # FORBIDDEN
raise InvalidToken(data['message']) raise InvalidToken(data['message'])
elif code == 400: elif code == 400:
@ -110,14 +113,6 @@ class Piexel:
fields = self._get_request([], [], **params) fields = self._get_request([], [], **params)
return self._get_response('indexer/paths', fields) return self._get_response('indexer/paths', fields)
def get_actors(self, **params):
"""
Récupère les acteurs
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'name', 'imdb_id', 'tmdb_id', 'limit', 'first', 'films', 'series'], [], **params)
return self._get_response('actors', fields)
def get_files(self, **params): def get_files(self, **params):
""" """
Récupère les fichiers Récupère les fichiers
@ -182,14 +177,6 @@ class Piexel:
fields = self._get_request(['season', 'episode', 'serie_id'], ['season', 'episode', 'serie_id'], **params) fields = self._get_request(['season', 'episode', 'serie_id'], ['season', 'episode', 'serie_id'], **params)
return self._get_response('episodes', fields, 'post') return self._get_response('episodes', fields, 'post')
def post_actor(self, **params):
"""
Ajoute un acteur
:param params: paramètres à passer
"""
fields = self._get_request(['name'], ['name'], **params)
return self._get_response('episodes', fields, 'post')
def post_broken(self, **params): def post_broken(self, **params):
""" """
Ajoute un lien mort Ajoute un lien mort
@ -238,14 +225,6 @@ class Piexel:
fields = self._get_request(['id', 'title', 'serid_id', 'tmdb_id', 'summary', 'cover_img', 'season', 'episode', 'rating', 'release_date'], ['id'], **params) fields = self._get_request(['id', 'title', 'serid_id', 'tmdb_id', 'summary', 'cover_img', 'season', 'episode', 'rating', 'release_date'], ['id'], **params)
return self._get_response('episodes', fields, 'put') return self._get_response('episodes', fields, 'put')
def put_actor(self, **params):
"""
Edite un acteur
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'name', 'imdb_id', 'tmdb_id', 'cover_img'], ['id'], **params)
return self._get_response('actors', fields, 'put')
def put_broken(self, **params): def put_broken(self, **params):
""" """
Edite un lien mort Edite un lien mort
@ -294,14 +273,6 @@ class Piexel:
fields = self._get_request(['id'], ['id'], **params) fields = self._get_request(['id'], ['id'], **params)
return self._get_response('episodes', fields, 'delete') return self._get_response('episodes', fields, 'delete')
def delete_actor(self, **params):
"""
Supprime un acteur
:param params: paramètres à passer
"""
fields = self._get_request(['id'], ['id'], **params)
return self._get_response('actors', fields, 'delete')
def delete_broken(self, **params): def delete_broken(self, **params):
""" """
Supprime un lien mort Supprime un lien mort

23
tokenizer.py
View file

@ -2,7 +2,6 @@
import piexel import piexel
import re import re
class Tokenizer: class Tokenizer:
def __init__(self, conf, api): def __init__(self, conf, api):
self.conf = conf self.conf = conf
@ -17,22 +16,22 @@ class Tokenizer:
def get_tokens_step(self, step): def get_tokens_step(self, step):
return [t for t in self.tk if t['step'] == step] return [t for t in self.tk if t['step'] == step]
def tokenise(self, filename): def tokenize(self, filename):
found = {} found = {'lang':[], 'quality':[], 'subtitle':[]}
for step in self.steps: for step in self.steps:
for tok in self.get_tokens_step(step): for tok in self.get_tokens_step(step):
if(not bool(tk['case_sensitive'])): if(not bool(int(tok['case_sensitive']))):
reg = re.compile(tok['token'], re.IGNORECASE) reg = re.compile(tok['token'], re.IGNORECASE)
else: else:
reg = re.compile(tok['token']) reg = re.compile(tok['token'])
if reg.match(filename): if reg.search(filename):
if(tok['lang']): for tok_lang in tok['languages']:
found['lang'] = tok['lang'] found['lang'].append(tok_lang['value'])
if(tok['quality']): for tok_qual in tok['qualities']:
found['quality'] = tok['quality'] found['quality'].append(tok_qual['value'])
if(tok['subtitle']): for tok_sub in tok['subtitle_languages']:
found['subtitles'] = tok['subtitle'] found['subtitle'].append(tok_sub['value'])
reg.sub(' ', filename) filename = reg.sub(' ', filename)
return filename, found return filename, found