Skip to content

Commit 7c00440

Browse files
RaNaNRaNaN
committed
fix in unrar plugin
1 parent 8766739 commit 7c00440

6 files changed

Lines changed: 135 additions & 68 deletions

File tree

‎module/Api.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ def statusDownloads(self):
240240

241241
return data
242242

243-
def addPackage(self, name, links, dest):
243+
def addPackage(self, name, links, dest=Destination.Queue):
244244
"""Adds a package, with links to desired destination.
245245
246246
:param name: name of the new package
@@ -343,6 +343,19 @@ def generateAndAddPackages(self, links, dest=Destination.Queue):
343343
return [self.addPackage(name, urls, dest) for name, urls
344344
in self.generatePackages(links).iteritems()]
345345

346+
def checkAndAddPackages(self, links, dest=Destination.Queue):
347+
"""Checks online status, retrieves names, and will add packages.\
348+
Because of this packages are not added immediatly, only for internal use.
349+
350+
:param links: list of urls
351+
:param dest: `Destination`
352+
:return: None
353+
"""
354+
data = self.core.pluginManager.parseUrls(urls)
355+
self.core.threadManager.createResultThread(data, True)
356+
357+
358+
346359
def getPackageData(self, pid):
347360
"""Returns complete information about package, and included files.
348361

‎module/PluginThread.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ def run(self):
407407

408408

409409
class InfoThread(PluginThread):
410-
def __init__(self, manager, data, pid=-1, rid=-1):
410+
def __init__(self, manager, data, pid=-1, rid=-1, add=False):
411411
"""Constructor"""
412412
PluginThread.__init__(self, manager)
413413

@@ -416,6 +416,7 @@ def __init__(self, manager, data, pid=-1, rid=-1):
416416
# [ .. (name, plugin) .. ]
417417

418418
self.rid = rid #result id
419+
self.add = add #add packages instead of return result
419420

420421
self.cache = [] #accumulated data
421422

@@ -440,6 +441,29 @@ def run(self):
440441
self.fetchForPlugin(pluginname, plugin, urls, self.updateDB)
441442
self.m.core.files.save()
442443

444+
elif self.add:
445+
for pluginname, urls in plugins.iteritems():
446+
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
447+
if hasattr(plugin, "getInfo"):
448+
self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True)
449+
450+
else:
451+
#generate default result
452+
result = [(url, 0, 3, url) for url in urls]
453+
454+
self.updateCache(pluginname, result)
455+
456+
457+
packs = parseNames([(name, url) for name, x,y, url in self.cache])
458+
459+
self.m.core.log.debug("Fetched and generated %d packages" % len(packs))
460+
461+
for k, v in packs:
462+
self.m.core.api.addPackage(k, v)
463+
464+
#empty cache
465+
del self.cache[:]
466+
443467
else: #post the results
444468

445469
self.m.infoResults[self.rid] = {}
@@ -489,6 +513,9 @@ def updateResult(self, plugin, result, force=False):
489513

490514
self.cache = []
491515

516+
def updateCache(self, plugin, result):
517+
self.cache.extend(result)
518+
492519
def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None):
493520
try:
494521
result = [] #result loaded from cache

‎module/ThreadManager.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,14 @@ def createInfoThread(self, data, pid):
8888
PluginThread.InfoThread(self, data, pid)
8989

9090
@lock
91-
def createResultThread(self, data):
91+
def createResultThread(self, data, add=False):
9292
""" creates a thread to fetch online status, returns result id """
9393
self.timestamp = time() + 5 * 60
9494

9595
rid = self.resultIDs
9696
self.resultIDs += 1
9797

98-
PluginThread.InfoThread(self, data, rid=rid)
98+
PluginThread.InfoThread(self, data, rid=rid, add=add)
9999

100100
return rid
101101

‎module/common/APIExerciser.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44
from threading import Thread
55
from random import choice, random, sample, randint
66
from time import time, sleep
7+
from math import floor
8+
import gc
79

810
from traceback import print_exc, format_exc
911

10-
from module.remote.thriftbackend.ThriftClient import ThriftClient
12+
from module.remote.thriftbackend.ThriftClient import ThriftClient, Destination
1113

1214
def createURLs():
1315
""" create some urls, some may fail """
@@ -24,6 +26,9 @@ def createURLs():
2426
AVOID = (0,3,8)
2527

2628
class APIExerciser(Thread):
29+
30+
idPool = 0
31+
2732
def __init__(self, core, thrift=False):
2833
Thread.__init__(self)
2934
self.setDaemon(True)
@@ -37,6 +42,12 @@ def __init__(self, core, thrift=False):
3742
else:
3843
self.api = core.api
3944

45+
46+
self.id = self.idPool
47+
48+
self.core.log.info("API Excerciser started %d" % self.id)
49+
APIExerciser.idPool += 1
50+
4051
self.start()
4152

4253
def run(self):
@@ -49,15 +60,17 @@ def run(self):
4960
try:
5061
self.testAPI()
5162
except Exception:
63+
self.core.log.error("Excerciser %d throw an execption" % self.id)
5264
print_exc()
5365
out.write(format_exc() + 2 * "\n")
5466
out.flush()
5567

5668
if not self.count % 100:
57-
print "Tested %s api calls" % self.count
69+
self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count))
5870
if not self.count % 1000:
5971
out.write("Tested %s api calls\n" % self.count)
6072
out.flush()
73+
self.core.log.info("Collected garbage: %d" % gc.collect())
6174

6275

6376
#sleep(random() / 500)
@@ -82,7 +95,7 @@ def addPackage(self):
8295
name = "".join(sample(string.ascii_letters, 10))
8396
urls = createURLs()
8497

85-
self.api.addPackage(name, urls, 0)
98+
self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector]))
8699

87100

88101
def deleteFiles(self):
@@ -98,12 +111,12 @@ def deleteFiles(self):
98111

99112

100113
def deletePackages(self):
101-
info = self.api.getQueue()
114+
info = choice([self.api.getQueue(), self.api.getCollector()])
102115
if not info: return
103116

104117
pids = [p.pid for p in info]
105118
if len(pids):
106-
pids = sample(pids, randint(1, max(len(pids) / 2, 1)))
119+
pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1)))
107120
self.api.deletePackages(pids)
108121

109122
def getFileData(self):

‎module/plugins/hooks/UnRar.py‎

Lines changed: 72 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,31 @@
2424
from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError
2525
from traceback import print_exc
2626

27+
import os
2728
from os.path import exists, join, isabs, isdir
2829
from os import remove, makedirs, rmdir, listdir, chown, chmod
29-
from pwd import getpwnam
30+
31+
if os.name != "nt":
32+
from pwd import getpwnam
33+
3034
import re
3135

3236
class UnRar(Hook):
3337
__name__ = "UnRar"
3438
__version__ = "0.11"
3539
__description__ = """unrar"""
36-
__config__ = [ ("activated", "bool", "Activated", False),
37-
("fullpath", "bool", "extract full path", True),
38-
("overwrite", "bool", "overwrite files", True),
39-
("passwordfile", "str", "unrar password file", "unrar_passwords.txt"),
40-
("deletearchive", "bool", "delete archives when done", False),
41-
("ramwarning", "bool", "warn about low ram", True),
42-
("renice", "int", "Cpu Priority", 10),
43-
("unrar_destination", "str", "Unpack files to", "")]
40+
__config__ = [("activated", "bool", "Activated", False),
41+
("fullpath", "bool", "extract full path", True),
42+
("overwrite", "bool", "overwrite files", True),
43+
("passwordfile", "str", "unrar password file", "unrar_passwords.txt"),
44+
("deletearchive", "bool", "delete archives when done", False),
45+
("ramwarning", "bool", "warn about low ram", True),
46+
("renice", "int", "Cpu Priority", 10),
47+
("unrar_destination", "str", "Unpack files to", "")]
4448
__threaded__ = ["packageFinished"]
4549
__author_name__ = ("mkaay")
4650
__author_mail__ = ("mkaay@mkaay.de")
47-
51+
4852
def setup(self):
4953
self.comments = ["# one password each line"]
5054
self.passwords = []
@@ -71,31 +75,31 @@ def setup(self):
7175
self.ram = 0
7276

7377
self.ram /= 1024
74-
75-
def setOwner(self,d,uid,gid,mode):
78+
79+
def setOwner(self, d, uid, gid, mode):
7680
if not exists(d):
77-
self.core.log.debug(_("Directory %s does not exist!") % d)
78-
return
79-
fileList=listdir(d)
80-
for fileEntry in fileList:
81-
fullEntryName=join(d,fileEntry)
82-
if isdir(fullEntryName):
83-
self.setOwner(fullEntryName,uid,gid,mode)
84-
try:
85-
chown(fullEntryName,uid,gid)
86-
chmod(fullEntryName,mode)
87-
except:
88-
self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName)
89-
self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
90-
continue
81+
self.core.log.debug(_("Directory %s does not exist!") % d)
82+
return
83+
84+
for fileEntry in listdir(d):
85+
fullEntryName = join(d, fileEntry)
86+
if isdir(fullEntryName):
87+
self.setOwner(fullEntryName, uid, gid, mode)
88+
try:
89+
chown(fullEntryName, uid, gid)
90+
chmod(fullEntryName, mode)
91+
except:
92+
self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName)
93+
self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
94+
continue
9195
try:
92-
chown(d,uid,gid)
93-
chmod(d,mode)
96+
chown(d, uid, gid)
97+
chmod(d, mode)
9498
except:
95-
self.core.log.debug(_("Chown/Chmod for %s failed") % d)
96-
self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
97-
return
98-
99+
self.core.log.debug(_("Chown/Chmod for %s failed") % d)
100+
self.core.log.debug(_("Exception: %s") % sys.exc_info()[0])
101+
return
102+
99103
def addPassword(self, pws):
100104
if not type(pws) == list: pws = [pws]
101105
pws.reverse()
@@ -105,14 +109,14 @@ def addPassword(self, pws):
105109
self.passwords.insert(0, pw)
106110

107111
with open(self.getConfig("passwordfile"), "w") as f:
108-
f.writelines([c+"\n" for c in self.comments])
109-
f.writelines([p+"\n" for p in self.passwords])
110-
112+
f.writelines([c + "\n" for c in self.comments])
113+
f.writelines([p + "\n" for p in self.passwords])
114+
111115
def removeFiles(self, pack, fname):
112116
if not self.getConfig("deletearchive"):
113117
return
114118
m = self.re_splitfile.search(fname)
115-
119+
116120
download_folder = self.core.config['general']['download_folder']
117121
if self.core.config['general']['folder_per_package']:
118122
folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding()))
@@ -124,11 +128,11 @@ def removeFiles(self, pack, fname):
124128
if nre.match(data["name"]):
125129
remove(join(folder, data["name"]))
126130
elif not m and fname.endswith(".rar"):
127-
nre = re.compile("^%s\.r..$" % fname.replace(".rar",""))
131+
nre = re.compile("^%s\.r..$" % fname.replace(".rar", ""))
128132
for fid, data in pack.getChildren().iteritems():
129133
if nre.match(data["name"]):
130134
remove(join(folder, data["name"]))
131-
135+
132136
def packageFinished(self, pack):
133137
if pack.password and pack.password.strip() and pack.password.strip() != "None":
134138
self.addPassword(pack.password.splitlines())
@@ -137,15 +141,16 @@ def packageFinished(self, pack):
137141
for fid, data in pack.getChildren().iteritems():
138142
m = self.re_splitfile.search(data["name"])
139143
if m and int(m.group(2)) == 1:
140-
files.append((fid,m.group(0)))
144+
files.append((fid, m.group(0)))
141145
elif not m and data["name"].endswith(".rar"):
142-
files.append((fid,data["name"]))
143-
146+
files.append((fid, data["name"]))
147+
144148
for fid, fname in files:
145149
self.core.log.info(_("starting Unrar of %s") % fname)
146150
pyfile = self.core.files.getFile(fid)
147151
pyfile.setStatus("processing")
148152
pyfile.progress.setRange(0, 100)
153+
149154
def s(p):
150155
pyfile.progress.setValue(p)
151156

@@ -170,12 +175,14 @@ def s(p):
170175

171176
self.core.log.debug(_("Destination folder %s") % destination)
172177
if not exists(destination):
173-
self.core.log.info(_("Creating destination folder %s") % destination)
174-
makedirs(destination)
178+
self.core.log.info(_("Creating destination folder %s") % destination)
179+
makedirs(destination)
175180

176-
u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice"))
181+
u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"),
182+
ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice"))
177183
try:
178-
success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, destination=destination, fullPath=self.getConfig("fullpath"))
184+
success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True,
185+
destination=destination, fullPath=self.getConfig("fullpath"))
179186
except WrongPasswordError:
180187
self.core.log.info(_("Unrar of %s failed (wrong password)") % fname)
181188
continue
@@ -195,7 +202,8 @@ def s(p):
195202
self.core.log.info(_("Unrar of %s failed") % fname)
196203
continue
197204
except LowRamError:
198-
self.log.warning(_("Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram)
205+
self.log.warning(_(
206+
"Your ram amount of %s MB seems not sufficient to unrar this file. You can deactivate this warning and risk instability") % self.ram)
199207
continue
200208
except UnknownError:
201209
if self.core.debug:
@@ -207,19 +215,24 @@ def s(p):
207215
self.core.log.info(_("Unrar of %s ok") % fname)
208216
self.removeFiles(pack, fname)
209217
if self.core.config['general']['folder_per_package']:
210-
if self.getConfig("deletearchive"):
211-
self.core.log.debug(_("Deleting package directory %s...") % folder)
212-
rmdir(folder)
213-
self.core.log.debug(_("Package directory %s has been deleted.") % folder)
214-
ownerUser=self.core.config['permission']['user']
215-
uinfo=getpwnam(ownerUser)
216-
fileMode=int(self.core.config['permission']['file'],8)
217-
self.core.log.debug(_("Setting destination file/directory owner to %s.") % ownerUser)
218-
self.core.log.debug(_("Setting destination file/directory mode to %s.") % fileMode)
219-
self.core.log.debug(_("Uid is %s.") % uinfo.pw_uid)
220-
self.core.log.debug(_("Gid is %s.") % uinfo.pw_gid)
221-
self.setOwner(destination,uinfo.pw_uid,uinfo.pw_gid,fileMode)
222-
self.core.log.debug(_("The owner/rights have been successfully changed."))
218+
if self.getConfig("deletearchive"):
219+
self.core.log.debug(_("Deleting package directory %s...") % folder)
220+
rmdir(folder)
221+
self.core.log.debug(_("Package directory %s has been deleted.") % folder)
222+
223+
if os.name != "nt" and self.core.config['permission']['change_dl'] and\
224+
self.core.config['permission']['change_file']:
225+
ownerUser = self.core.config['permission']['user']
226+
fileMode = int(self.core.config['permission']['file'], 8)
227+
228+
self.core.log.debug("Setting destination file/directory owner / mode to %s / %s"
229+
% (ownerUser, fileMode))
230+
231+
uinfo = getpwnam(ownerUser)
232+
self.core.log.debug("Uid/Gid is %s/%s." % (uinfo.pw_uid, uinfo.pw_gid))
233+
self.setOwner(destination, uinfo.pw_uid, uinfo.pw_gid, fileMode)
234+
self.core.log.debug("The owner/rights have been successfully changed.")
235+
223236
self.core.hookManager.unrarFinished(folder, fname)
224237
else:
225238
self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname)

0 commit comments

Comments
 (0)