piexel/piexel-indexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add indexed files

Browse source
This commit is contained in:
bdvllrs 2018-02-06 14:31:01 +01:00
parent cbbb8148b5
commit bf1fa8e3dd
7 changed files with 250 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -3,3 +3,4 @@ __pycache__/*
*.sublime-project
*.sublime-workspace
config.conf
.idea/*

2
config.py
View file

@ -12,6 +12,7 @@ class Config:
"""
Gère le fichier de configuration
"""
def __init__(self):
try:
@ -51,4 +52,3 @@ class Config:
def is_valid_file(self, name):
_, ext = posixpath.splitext(name)
return ext[1:] in self.obj['api']['extensions']

3
file.py
View file

@ -2,10 +2,12 @@
import posixpath
import re
class File:
"""
Décrit une référence de fichier dans le disque
"""
def __init__(self, path, name, info, api_id=None, api_fileid=None, api_fileable_type=None):
self.path = path
self.name = name
@ -97,4 +99,3 @@ class File:
def __repr__(self):
return str(self)

2
filerule.py
View file

@ -2,6 +2,7 @@ import re
PATTERN_BASIC = re.compile(r'#(.+?)#')
def match_rules(filename, rules):
for r in rules:
match = r.apply(filename)
@ -9,6 +10,7 @@ def match_rules(filename, rules):
return r, match
return None
class FileRule:
def __init__(self, rule, conf):
# trouve les marqueurs dans la règle

39
main.py
View file

@ -43,6 +43,7 @@ def ftpwalk(directory, ftp):
# mais retourne les résultats intermédiaires
yield (current, Ldirs, Lfiles)
def visit_folder(domain, api, rules, tok):
"""
Visite un dossier sur un serveur, spécifié par <domain>,
@ -60,15 +61,22 @@ def visit_folder(domain, api, rules, tok):
L_moved = [] # fichiers déplacés sur le serveur FTP
# Lecture des fichiers sur le serveur FTP
Lloc = []
indexed_files = []
for path, _, files in ftpwalk(domain['path'], ftp):
# Ajoute les fichiers correspondants aux extensions
for f in files:
indexed_files.append({
"path": domain['server'] + path,
"name": f
})
match = filerule.match_rules(path + '/' + f, rules)
if match:
# print('got match:',match[1], 'name:',path+'/'+f)
F = file.File(path, f, match[1])
F.extract_title(tok)
Lloc.append(F)
print('indexing all files :', len(indexed_files), ' files')
api.post_index(indexed_files)
ftp.close()
print('total loc for ', domain['server'] + domain['path'], ':', len(Lloc))
# Récupère les fichiers de l'api
@ -80,7 +88,8 @@ def visit_folder(domain, api, rules, tok):
else:
year = int(info['filable']['release_date'][:4])
nfo['YEAR'] = year
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'], api_fileid=info['id'], api_fileable_type=info['filable_type'])
F = file.File(info['path'][len(domain['server']):], info['name'], nfo, api_id=info['filable_id'],
api_fileid=info['id'], api_fileable_type=info['filable_type'])
F.extract_title(tok)
match = filerule.match_rules(F.path + '/' + F.name, rules)
if match:
@ -101,6 +110,7 @@ def visit_folder(domain, api, rules, tok):
print('visit finished ', domain['server'] + domain['path'])
def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
"""
Utilise les listes des fichiers locaux Lfilm_loc et des fichiers de l'API Lfilm_api
@ -115,7 +125,9 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
# print('api titles:', '|'.join(sorted([f.title for f in Lapi])))
# supprime les dossiers de l'api (ils ne devraient pas apparaître)
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules))
Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
@ -162,7 +174,8 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
Lunref.remove(f)
print('invalid:' + '\n'.join(str(f.api_fileid) + ' ' + str(f) for f in Linvalid))
print('missing (', len(Lmissing), '):','\n'.join([str(e.api_id)+':'+repr(e)+'('+e.title+')' for e in sorted(Lmissing, key=lambda e:e.title)]))
print('missing (', len(Lmissing), '):', '\n'.join(
[str(e.api_id) + ':' + repr(e) + '(' + e.title + ')' for e in sorted(Lmissing, key=lambda e: e.title)]))
print('unreferenced:' + '\n'.join(str(f) for f in sorted(Lunref, key=lambda e: e.title)))
# print('unreferenced titles:\n', '\n'.join(sorted([f.title for f in Lunref])))
@ -225,7 +238,8 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
i += 1
print('[' + str(i) + '/' + str(len(Llink2)) + ']' + 'link2:', film.title)
try:
resp = api.post_file(path=domain['server']+film.path, name=film.name, type='film', type_id=filmID, **film.additional_info())
resp = api.post_file(path=domain['server'] + film.path, name=film.name, type='film', type_id=filmID,
**film.additional_info())
if 'id' in resp:
post_markers(api, film, resp['id'])
time.sleep(1)
@ -268,6 +282,7 @@ def handle_films(Lfilm_loc, Lfilm_api, domain, api, rules, tok):
# TODO: traiter les films non postés (Lcannot_post)
def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
"""
Utilise les listes des fichiers locaux Lepisode_loc et des fichiers de l'API Lepisode_api
@ -277,7 +292,9 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
Lapi = Lepisode_api
# fichiers invalides
Lapi, Linvalid = filter_by(Lapi, lambda f:tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path+'/'+f.name, rules))
Lapi, Linvalid = filter_by(Lapi,
lambda f: tok.conf.is_valid_file(f.name) and filerule.match_rules(f.path + '/' + f.name,
rules))
# Compare avec la liste de l'api
Lmissing = [f for f in Lapi if f not in Lloc] # fichiers non présents localement
@ -364,7 +381,8 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
season_num = int(episode.info['SEASON_NUM'])
resp = api.post_episode(serie_id=serie_id, episode=episode_num, season=season_num)
if 'id' in resp:
resp = api.post_file(path=domain['server']+episode.path, name=episode.name, type='episode', type_id=resp["id"])
resp = api.post_file(path=domain['server'] + episode.path, name=episode.name, type='episode',
type_id=resp["id"])
print('response: ', resp)
else:
print('episode not posted:', resp)
@ -376,6 +394,7 @@ def handle_episodes(Lepisode_loc, Lepisode_api, domain, api, rules, tok):
# TODO: traiter les séries non référencées (series_not_found)
def filter_by(L, f_prop):
"""
Sépare la liste L en deux listes,
@ -389,6 +408,7 @@ def filter_by(L, f_prop):
prop_false.append(e)
return prop_true, prop_false
def update_find_by_common(L1, L2, f_prop):
"""
Effectue une recherche de propriétés commune entre les éléments
@ -402,6 +422,7 @@ def update_find_by_common(L1, L2, f_prop):
L2 = [e for e in L2 if e not in found2]
return L1, L2, found
def find_by_common(L1, L2, f_prop):
"""
Associe les éléments de <L1> et <L2> à travers leur propriétés
@ -435,6 +456,7 @@ def post_markers(api, file_, fileid):
api.post_file_subtitle(fileid, value=sub)
time.sleep(1)
def main():
"""
Fonction principale du programme, réalise toute les opérations
@ -449,9 +471,10 @@ def main():
rules = api.get_paths()
for fold in folders[1:]:
applicable = [filerule.FileRule(re.escape(fold['path'])+'\\/'+r['regex'], conf) for r in rules if int(r['indexer_folder_id']) == fold['id']]
applicable = [filerule.FileRule(re.escape(fold['path']) + '\\/' + r['regex'], conf) for r in rules if
int(r['indexer_folder_id']) == fold['id']]
visit_folder(fold, api, applicable, tokens)
if __name__ == '__main__':
main()

47
piexel.py
View file

@ -1,19 +1,25 @@
# coding:utf-8
import requests
from requests.auth import HTTPBasicAuth
class PiexelErrors(Exception):
pass
class InvalidToken(PiexelErrors):
pass
class ParameterError(PiexelErrors):
pass
class InvalidResponse(PiexelErrors):
pass
class Piexel:
def __init__(self, domain, app='', token='', endpoint='/api/'):
# self.app = app
@ -22,7 +28,7 @@ class Piexel:
self.domain = domain
self.endpoint = endpoint
def _get_response(self, controller, fields, request_type='get'):
def _get_response(self, controller, fields, request_type='get', data=None):
"""
Build response
:param controller: controller à utiliser
@ -32,8 +38,14 @@ class Piexel:
if request_type == 'get':
response = requests.get(url, auth=self.auth, params=fields)
elif request_type == 'post':
if data is not None:
response = requests.post(url, auth=self.auth, params=fields, json=data)
else:
response = requests.post(url, auth=self.auth, params=fields)
elif request_type == 'put':
if data is not None:
response = requests.put(url, auth=self.auth, params=fields, json=data)
else:
response = requests.put(url, auth=self.auth, params=fields)
elif request_type == 'delete':
response = requests.delete(url, auth=self.auth, data=fields)
@ -68,7 +80,8 @@ class Piexel:
Récupère les films
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [], **params)
fields = self._get_request(['id', 'title', 'title_vo', 'imdb_id', 'limit', 'first', 'first', 'files'], [],
**params)
return self._get_response('films', fields)
def get_series(self, **params):
@ -84,7 +97,8 @@ class Piexel:
Récupère les épisodes
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [], **params)
fields = self._get_request(['id', 'serie_id', 'title', 'imdb_id', 'limit', 'first', 'first', 'episodes'], [],
**params)
return self._get_response('episodes', fields)
def get_tokens(self, **params):
@ -180,7 +194,8 @@ class Piexel:
Ajoute un sous-titre
:param params: paramètres à passer
"""
fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'], ['path', 'name', 'type', 'type_id'], **params)
fields = self._get_request(['path', 'name', 'type', 'type_id', 'quality', 'lang'],
['path', 'name', 'type', 'type_id'], **params)
return self._get_response('subtitles', fields, 'post')
def post_serie(self, **params):
@ -204,7 +219,8 @@ class Piexel:
Edite un fichier
:param params: paramètres à passer
"""
fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [], **params)
fields = self._get_request(['path', 'name', 'filable_type', 'filable_id', 'quality', 'lang', 'subtitles'], [],
**params)
return self._get_response('file/' + str(id), fields, 'put')
def delete_file(self, id, **params):
@ -215,3 +231,24 @@ class Piexel:
fields = self._get_request([], [], **params)
return self._get_response('file/' + str(id), fields, 'delete')
def get_index(self, **params):
"""
Récupère les fichiers indexés
:param params: paramètres à passer
"""
fields = self._get_request(['id', 'path', 'name', 'limit', 'first', 'is_referenced', 'like'], [], **params)
return self._get_response('index', fields)
def post_index(self, params):
"""
Ajoute un fichier
:param params: paramètres à passer
"""
return self._get_response('index', {}, 'post', params)
def delete_index(self, id):
"""
Supprime un fichier indéxé
:param id: id du fichier à supprimer
"""
return self._get_response('index/' + str(id), {}, 'delete')

6
tokenizer.py
View file

@ -2,6 +2,7 @@
import piexel
import re
class Tokenizer:
def __init__(self, conf, api):
self.conf = conf
@ -46,8 +47,3 @@ class Tokenizer:
for typ in found:
found[typ] = [e for e in found[typ] if e != 'N/A']
return filename, found