From: Ira W. Snyder Date: Mon, 1 Jan 2007 08:36:14 +0000 (-0800) Subject: Major Projectwide Restructuring X-Git-Tag: v2.0.0~16 X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=commitdiff_plain;h=76892bd66237df907e45dc5ad8237ae58adc49ae Major Projectwide Restructuring Some major deficiencies were noticed in the previous design. Specifically, the order of execution of steps was not customizable on a per-type basis. This led to the problem of some types not being able to finish correctly. Since this was obviously sub-optimal, the change was needed. This adds a RarslaveDetector class, and many PAR2Set-derived classes. Each of the PAR2Set-derived classes will be able to detect and fully verify / repair / extract their type of set. When adding new sets, the new types only need to be added to the RarslaveDetector class. Everything else is automatic. Signed-off-by: Ira W. Snyder --- diff --git a/PAR2Set.py b/PAR2Set.py new file mode 100644 index 0000000..323fc24 --- /dev/null +++ b/PAR2Set.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +from RarslaveCommon import * +import RarslaveLogger +import Par2Parser + +import re +import os + +# This is a fairly generic class which does all of the major things that a PAR2 +# set will need to have done to be verified and extracted. For most "normal" types +# you won't need to override hardly anything. +# +# It is ok to override other functions if the need arises, just make sure that you +# understand why things are done the way that they are in the original versions. +# +# Assumptions made about each of the run*() functions: +# ============================================================================== +# The state of self.name_matched_files and self.prot_matched_files will be consistent +# with the real, in-filesystem state at the time that they are called. +# (This is why runAll() calls update_matches() all the time.) +# +# Required overrides: +# ============================================================================== +# find_extraction_heads () +# extraction_function () +# + +class PAR2Set (object): + + # Instance Variables + # ========================================================================== + # dir -- The directory this set lives in + # p2file -- The starting PAR2 file + # basename -- The basename of the set, guessed from the PAR2 file + # all_p2files -- All PAR2 files of the set, guessed from the PAR2 file name only + # name_matched_files -- Files in this set, guessed by name only + # prot_matched_files -- Files in this set, guessed by parsing the PAR2 only + + def __init__ (self, dir, p2file): + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, p2file)) + + # The real "meat" of the class + self.dir = dir + self.p2file = p2file + self.basename = get_basename (p2file) + + # Find files that match by name only + self.name_matched_files = find_name_matches (self.dir, self.basename) + + # Find all par2 files for this set using name matches + self.all_p2files = find_par2_files (self.name_matched_files) + + # Try to get the protected files for this set + self.prot_matched_files = parse_all_par2 (self.dir, self.p2file, self.all_p2files) + + # Setup the all_files combined set (for convenience only) + self.all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + def __eq__ (self, rhs): + return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ + list_eq (self.name_matched_files, rhs.name_matched_files) and \ + list_eq (self.prot_matched_files, rhs.prot_matched_files) + + def update_matches (self): + """Updates the contents of instance variables which are likely to change after + running an operation, usually one which will create new files.""" + + self.name_matched_files = find_name_matches (self.dir, self.basename) + self.all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + def runVerifyAndRepair (self): + PAR2_CMD = config_get_value ('commands', 'par2repair') + + # assemble the command + # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] + command = "%s \"%s\" " % (PAR2_CMD, self.p2file) + + for f in self.all_p2files: + if f != self.p2file: + command += "\"%s\" " % os.path.split (f)[1] + + # run the command + ret = run_command (command, self.dir) + + # check the result + if ret != 0: + fatalMessage ('PAR2 Check / Repair failed: %s' % self.p2file) + return -ECHECK + + return SUCCESS + + def find_deleteable_files (self): + DELETE_REGEX = config_get_value ('regular expressions', 'delete_regex') + dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + + return [f for f in self.all_files if dregex.match (f)] + + def delete_list_of_files (self, dir, files, interactive=False): + # Delete a list of files + + assert os.path.isdir (dir) + + done = False + valid_y = ['Y', 'YES'] + valid_n = ['N', 'NO', ''] + + if interactive: + while not done: + print 'Do you want to delete the following?:' + for f in files: + print f + s = raw_input ('Delete [y/N]: ').upper() + + if s in valid_y + valid_n: + done = True + + if s in valid_n: + return SUCCESS + + for f in files: + try: + os.remove (os.path.join (dir, f)) + debugMessage ('Deleteing: %s' % os.path.join (dir, f)) + except: + fatalMessage ('Failed to delete: %s' % os.path.join (dir, f)) + return -EDELETE + + return SUCCESS + + def runDelete (self): + deleteable_files = self.find_deleteable_files () + ret = self.delete_list_of_files (self.dir, deleteable_files, \ + options_get_value ('interactive')) + + return ret + + def runAll (self): + + # Repair Stage + ret = self.runVerifyAndRepair () + + if ret != SUCCESS: + fatalMessage ('Repair stage failed for: %s' % self.p2file) + return -ECHECK + + self.update_matches () + + # Extraction Stage + ret = self.runExtract () + + if ret != SUCCESS: + fatalMessage ('Extraction stage failed for: %s' % self.p2file) + return -EEXTRACT + + self.update_matches () + + # Deletion Stage + ret = self.runDelete () + + if ret != SUCCESS: + fatalMessage ('Deletion stage failed for: %s' % self.p2file) + return -EDELETE + + normalMessage ('Successfully completed: %s' % self.p2file) + return SUCCESS + + def safe_create_directory (self, dir): + if dir == None: + return SUCCESS + + if os.path.isdir (dir): + return SUCCESS + + try: + os.makedirs (dir) + verboseMessage ('Created directory: %s' % dir) + except OSError: + fatalMessage ('FAILED to create directory: %s' % dir) + return -ECREATE + + return SUCCESS + + def runExtract (self, todir=None): + """Extract all heads of this set""" + + # Extract to the head's dir if we don't care where to extract + if todir == None: + todir = self.dir + + # Create the directory $todir if it doesn't exist + ret = self.safe_create_directory (todir) + + if ret != SUCCESS: + return -EEXTRACT + + # Call the extraction function on each head + for h in self.find_extraction_heads (): + full_head = full_abspath (os.path.join (self.dir, h)) + ret = self.extraction_function (full_head, todir) + debugMessage ('Extraction Function returned: %d' % ret) + + # Check error code + if ret != SUCCESS: + fatalMessage ('Failed extracting: %s' % h) + return -EEXTRACT + + return SUCCESS + + def find_extraction_heads (self): + assert False # You MUST override this on a per-type basis + + def extraction_function (self, file, todir): + # NOTE: Please keep the prototype the same for all overridden functions. + # Doing so will guarantee that your life is made much easier. + # + # Also note that the todir given will always be valid for the current directory + # when the function is called. + + assert False # You MUST override this on a per-type basis + diff --git a/PAR2Set_EF_NEWRAR.py b/PAR2Set_EF_NEWRAR.py new file mode 100644 index 0000000..6b90591 --- /dev/null +++ b/PAR2Set_EF_NEWRAR.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set_NEWRAR +import PAR2Set_EXTRACTFIRST +from RarslaveCommon import * + +# +# This is an old-style rar type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.rar +# X.r00 +# X.r01 +# +# OR +# +# ABC.rar +# ABC.r00 +# ABC.r01 +# +# Where the PAR2 files protect all files that do not match in basename +# with the PAR2 file itself. +# + +def detect_EF_NEWRAR (name_files, prot_files): + return has_a_match ('^.*\.part0*1\.rar$', name_files) \ + and not has_a_match ('^.*\.part0*1\.rar$', prot_files) + + +class PAR2Set_EF_NEWRAR (PAR2Set_EXTRACTFIRST.PAR2Set_EXTRACTFIRST, + PAR2Set_NEWRAR.PAR2Set_NEWRAR): + + def __repr__ (self): + return 'EXTRACTFIRST NEWRAR' + diff --git a/PAR2Set_EF_OLDRAR.py b/PAR2Set_EF_OLDRAR.py new file mode 100644 index 0000000..9363ca7 --- /dev/null +++ b/PAR2Set_EF_OLDRAR.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set_OLDRAR +import PAR2Set_EXTRACTFIRST +from RarslaveCommon import * + +# +# This is an old-style rar type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.rar +# X.r00 +# X.r01 +# +# OR +# +# ABC.rar +# ABC.r00 +# ABC.r01 +# +# Where the PAR2 files protect all files that do not match in basename +# with the PAR2 file itself. +# + +def detect_EF_OLDRAR (name_files, prot_files): + return has_a_match ('^.*\.r00$', name_files) \ + and not has_a_match ('^.*\.r00$', prot_files) + + +class PAR2Set_EF_OLDRAR (PAR2Set_EXTRACTFIRST.PAR2Set_EXTRACTFIRST, + PAR2Set_OLDRAR.PAR2Set_OLDRAR): + + def __repr__ (self): + return 'EXTRACTFIRST OLDRAR' + diff --git a/PAR2Set_EXTRACTFIRST.py b/PAR2Set_EXTRACTFIRST.py new file mode 100644 index 0000000..ab4b1f6 --- /dev/null +++ b/PAR2Set_EXTRACTFIRST.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set +from RarslaveCommon import * + +# +# This is another base class for types that must +# run the extraction routine before the repair routine +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.part01.rar +# X.part02.rar +# X.part03.rar +# +# Where the PAR2 files protect a file named X.avi, but not the X.part01.rar +# (and other) files. +# + +class PAR2Set_EXTRACTFIRST (PAR2Set.PAR2Set): + + def runAll (self): + + # Extraction Stage + ret = self.runExtract () + + if ret != SUCCESS: + fatalMessage ('Extraction stage failed for: %s' % self.p2file) + return -EEXTRACT + + self.update_matches () + + # Repair Stage + ret = self.runVerifyAndRepair () + + if ret != SUCCESS: + fatalMessage ('Repair stage failed for: %s' % self.p2file) + return -ECHECK + + self.update_matches () + + # Deletion Stage + ret = self.runDelete () + + if ret != SUCCESS: + fatalMessage ('Deletion stage failed for: %s' % self.p2file) + return -EDELETE + + normalMessage ('Successfully completed: %s' % self.p2file) + return SUCCESS + diff --git a/PAR2Set_JOIN.py b/PAR2Set_JOIN.py new file mode 100644 index 0000000..59b2c9b --- /dev/null +++ b/PAR2Set_JOIN.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set +from RarslaveCommon import * + +# +# This is a regular joined file type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.001 +# X.002 +# X.003 +# +# Where the PAR2 files protect a file named X.avi (or similar). It will not +# work where the PAR2 files are protecting the .001, etc files directly. +# + +def detect_JOIN (name_files, prot_files): + return has_a_match ('^.*\.\d\d\d$', name_files) + + +class PAR2Set_JOIN (PAR2Set.PAR2Set): + + def __repr__ (self): + return 'JOIN' + + def find_joinfiles (self): + return find_matches ('^.*\.\d\d\d$', self.name_matched_files) + + def runVerifyAndRepair (self): + PAR2_CMD = config_get_value ('commands', 'par2repair') + + # assemble the command + # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] + command = "%s \"%s\" " % (PAR2_CMD, self.p2file) + + for f in self.all_p2files: + if f != self.p2file: + command += "\"%s\" " % os.path.split (f)[1] + + for f in self.find_joinfiles (): + command += "\"%s\" " % os.path.split (f)[1] + + # run the command + ret = run_command (command, self.dir) + + # check the result + if ret != 0: + fatalMessage ('PAR2 Check / Repair failed: %s' % self.p2file) + return -ECHECK + + return SUCCESS + + def find_extraction_heads (self): + return self.prot_matched_files + + def extraction_function (self, file, todir): + """Extract a single file of this type to the given directory""" + + NOEXTRACT_CMD = config_get_value ('commands', 'noextract') + + # Make sure that both files are not the same file. If they are, don't run at all. + if os.path.samefile (file, os.path.join (todir, file)): + return SUCCESS + + cmd = NOEXTRACT_CMD % (file, todir) + ret = run_command (cmd) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + diff --git a/PAR2Set_NEWRAR.py b/PAR2Set_NEWRAR.py new file mode 100644 index 0000000..763c025 --- /dev/null +++ b/PAR2Set_NEWRAR.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set +from RarslaveCommon import * + +# +# This is an new-style rar type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.part01.rar +# X.part02.rar +# X.part03.rar +# +# OR +# +# ABC.part01.rar +# ABC.part02.rar +# ABC.part03.rar +# +# Where the PAR2 files protect all files that do not match in basename +# with the PAR2 file itself. +# + +def detect_NEWRAR (name_files, prot_files): + return has_a_match ('^.*\.part0*1\.rar$', prot_files) + + +class PAR2Set_NEWRAR (PAR2Set.PAR2Set): + + def __repr__ (self): + return 'NEWRAR' + + def find_extraction_heads (self): + return find_matches ('^.*\.part0*1\.rar$', self.all_files) + + def extraction_function (self, file, todir): + assert os.path.isfile (file) + assert os.path.isdir (todir) + + RAR_CMD = config_get_value ('commands', 'unrar') + + cmd = '%s \"%s\"' % (RAR_CMD, file) + ret = run_command (cmd, todir) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + diff --git a/PAR2Set_OLDRAR.py b/PAR2Set_OLDRAR.py new file mode 100644 index 0000000..5a099fe --- /dev/null +++ b/PAR2Set_OLDRAR.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set +from RarslaveCommon import * + +# +# This is an old-style rar type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.rar +# X.r00 +# X.r01 +# +# OR +# +# ABC.rar +# ABC.r00 +# ABC.r01 +# +# Where the PAR2 files protect all files that do not match in basename +# with the PAR2 file itself. +# + +def detect_OLDRAR (name_files, prot_files): + return has_a_match ('^.*\.r00$', prot_files) + + +class PAR2Set_OLDRAR (PAR2Set.PAR2Set): + + def __repr__ (self): + return 'OLDRAR' + + def find_extraction_heads (self): + return find_matches ('^.*\.rar', self.all_files) + + def extraction_function (self, file, todir): + assert os.path.isfile (file) + assert os.path.isdir (todir) + + RAR_CMD = config_get_value ('commands', 'unrar') + + cmd = '%s \"%s\"' % (RAR_CMD, file) + ret = run_command (cmd, todir) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + diff --git a/PAR2Set_ZIP.py b/PAR2Set_ZIP.py new file mode 100644 index 0000000..01b19f7 --- /dev/null +++ b/PAR2Set_ZIP.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import PAR2Set +from RarslaveCommon import * + +# +# This is a regular zip file type +# +# It will detect sets like the following: +# X.par2 +# X.vol0+1.par2 +# X.vol1+2.par2 +# X.zip +# ABC.zip +# +# Where the PAR2 files protect a file named X.zip and/or ABC.zip. +# + +def detect_ZIP (name_files, prot_files): + all_files = no_duplicates (name_files + prot_files) + return has_a_match ('^.*\.zip$', all_files) + + +class PAR2Set_ZIP (PAR2Set.PAR2Set): + + def __repr__ (self): + return 'ZIP' + + def find_extraction_heads (self): + return find_matches ('^.*\.zip', self.all_files) + + def extraction_function (self, file, todir): + ZIP_CMD = self.config_get_value ('commands', 'unzip') + + cmd = ZIP_CMD % (file, todir) + ret = run_command (cmd) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + diff --git a/RarslaveCommon.py b/RarslaveCommon.py new file mode 100644 index 0000000..1ed864c --- /dev/null +++ b/RarslaveCommon.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +__author__ = "Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006, Ira W. Snyder (devel@irasnyder.com)" +__license__ = "GNU GPL v2 (or, at your option, any later version)" + +# This module holds all of the common functions used throughout the rarslave project. + +import os +import re + +import RarslaveGlobals +import RarslaveLogger +import Par2Parser + +# Global constants +(SUCCESS, ECHECK, EEXTRACT, EDELETE, ECREATE, EDETECT, EPARSE) = range(7) + +def find_matches (regex, li, ignorecase=True): + + if ignorecase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) + + return [e for e in li if cregex.match (e)] + +def has_a_match (regex, li, ignorecase=True): + + if ignorecase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) + + for e in li: + if cregex.match (e): + return True + + return False + +def no_duplicates (li): + """Removes all duplicates from a list""" + return list(set(li)) + +def run_command (cmd, indir=None): + # Runs the specified command-line in the directory given (or, in the current directory + # if none is given). It returns the status code given by the application. + + pwd = os.getcwd () + + if indir != None: + assert os.path.isdir (indir) # MUST be a directory! + os.chdir (indir) + + ret = os.system (cmd) + os.chdir (pwd) + return ret + +def full_abspath (p): + return os.path.abspath (os.path.expanduser (p)) + +def find_par2_files (files): + """Find all par2 files in the list $files""" + + PAR2_REGEX = config_get_value ('regular expressions', 'par2_regex') + regex = re.compile (PAR2_REGEX, re.IGNORECASE) + return [f for f in files if regex.match (f)] + +def find_name_matches (dir, basename): + """Finds files which are likely to be part of the set corresponding + to $name in the directory $dir""" + + assert os.path.isdir (dir) + + ename = re.escape (basename) + regex = re.compile ('^%s.*$' % (ename, )) + + return [f for f in os.listdir (dir) if regex.match (f)] + +def parse_all_par2 (dir, p2head, p2files): + """Searches though p2files and tries to parse at least one of them""" + done = False + files = [] + + for f in p2files: + + # Exit early if we've found a good file + if done: + break + + try: + files = Par2Parser.get_protected_files (dir, f) + done = True + except (EnvironmentError, OSError, OverflowError): + verboseMessage ('Corrupt PAR2 file: %s' % f) + + # Now that we're out of the loop, check if we really finished + if not done: + fatalMessage ('All PAR2 files corrupt for: %s' % p2head) + + # Return whatever we've got, empty or not + return files + +def get_basename (name): + """Strips most kinds of endings from a filename""" + + regex = config_get_value ('regular expressions', 'basename_regex') + r = re.compile (regex, re.IGNORECASE) + done = False + + while not done: + done = True + + if r.match (name): + g = r.match (name).groups() + name = g[0] + done = False + + return name + +def list_eq (l1, l2): + + if len(l1) != len(l2): + return False + + for e in l1: + if e not in l2: + return False + + return True + +# Convience functions for the logger + +def fatalMessage (msg): + RarslaveGlobals.logger.addMessage (msg, RarslaveLogger.MessageType.Fatal) + +def normalMessage (msg): + RarslaveGlobals.logger.addMessage (msg, RarslaveLogger.MessageType.Normal) + +def verboseMessage (msg): + RarslaveGlobals.logger.addMessage (msg, RarslaveLogger.MessageType.Verbose) + +def debugMessage (msg): + RarslaveGlobals.logger.addMessage (msg, RarslaveLogger.MessageType.Debug) + +# Convience functions for the config + +def config_get_value (section, name): + return RarslaveGlobals.config.get_value (section, name) + +# Convience functions for the options + +def options_get_value (name): + return getattr (RarslaveGlobals.options, name) + +def main (): + pass + +if __name__ == '__main__': + main () + diff --git a/RarslaveDetector.py b/RarslaveDetector.py new file mode 100644 index 0000000..85316ca --- /dev/null +++ b/RarslaveDetector.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +from RarslaveCommon import * + +# PAR2Set-derived types +import PAR2Set_JOIN +import PAR2Set_ZIP +import PAR2Set_OLDRAR +import PAR2Set_NEWRAR +import PAR2Set_EF_OLDRAR +import PAR2Set_EF_NEWRAR + +class RarslaveDetector (object): + + # A tuple of tuples with the following definition: + # (TYPE_NAME, DETECTION_FUNCTION, PAR2Set-derived class) + + TYPES = ( (PAR2Set_JOIN.detect_JOIN, PAR2Set_JOIN.PAR2Set_JOIN), + (PAR2Set_ZIP.detect_ZIP, PAR2Set_ZIP.PAR2Set_ZIP), + (PAR2Set_OLDRAR.detect_OLDRAR, PAR2Set_OLDRAR.PAR2Set_OLDRAR), + (PAR2Set_NEWRAR.detect_NEWRAR, PAR2Set_NEWRAR.PAR2Set_NEWRAR), + (PAR2Set_EF_OLDRAR.detect_EF_OLDRAR, PAR2Set_EF_OLDRAR.PAR2Set_EF_OLDRAR), + (PAR2Set_EF_NEWRAR.detect_EF_NEWRAR, PAR2Set_EF_NEWRAR.PAR2Set_EF_NEWRAR), + ) + + def __init__ (self, dir, p2file): + + # The real "meat" of the class + self.dir = dir + self.p2file = p2file + self.basename = get_basename (p2file) + + # Find files that match by name only + self.name_matched_files = find_name_matches (self.dir, self.basename) + + # Find all par2 files for this set using name matches + self.all_p2files = find_par2_files (self.name_matched_files) + + # Try to get the protected files for this set + self.prot_matched_files = parse_all_par2 (self.dir, self.p2file, self.all_p2files) + + def runMatchingTypes (self): + # Now tries to run every type of PAR2Set-derived class for which the detector + # detects that the class is valid. + + detected = False + + for (detector, classname) in self.TYPES: + if detector (self.name_matched_files, self.prot_matched_files): + # The detector matched, so we're up and running! + + p2set = classname (self.dir, self.p2file) + + detected = True + debugMessage ('Detected type: %s' % p2set) + + # Try to have rarslave do it's thing + ret = p2set.runAll () + + # If something already worked, there is no need to continue, + # since we've already finished! + if ret == SUCCESS: + break + else: + fatalMessage ('Detected type failed for: %s' % self.p2file) + + # Make sure we detected at least one valid type + if not detected: + fatalMessage ('Unable to determine type: %s' % self.p2file) + verboseMessage ('The following information will help in writing a detector:') + verboseMessage ('name_matches: %s' % self.name_matched_files) + verboseMessage ('prot_matches: %s' % self.prot_matched_files) + return -EDETECT + + # Make sure that something worked + if ret != SUCCESS: + fatalMessage ('All types failed for: %s' % self.p2file) + + return ret + +def main (): + pass + +if __name__ == '__main__': + main () + diff --git a/RarslaveGlobals.py b/RarslaveGlobals.py new file mode 100644 index 0000000..32a4aeb --- /dev/null +++ b/RarslaveGlobals.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +import RarslaveLogger +import RarslaveConfig + +# I know that this sucks majorly, but I kinda need it to keep things +# sane in the code. I don't /want/ to have to keep passing the config, +# options, and logger around all the time. + +# This will hold the configuration from the configuration file. This should +# only be used to hold statically, non-runtime alterable content. +config = RarslaveConfig.RarslaveConfig () + +# This will hold the configuration from the command-line options. You should +# probably be using this to get values in most cases. It takes its defaults +# from the configuration file. +options = None + +# This will hold the logger that will be able to log everything in all the +# classes in rarslave. +logger = RarslaveLogger.RarslaveLogger () + diff --git a/rarslave.py b/rarslave.py index 0d25883..3f7ee54 100644 --- a/rarslave.py +++ b/rarslave.py @@ -4,478 +4,32 @@ VERSION="2.0.0" PROGRAM="rarslave2" -import re, os, sys, optparse -import Par2Parser -import RarslaveConfig +import os, sys, optparse import RarslaveLogger +import RarslaveDetector +import RarslaveGlobals +from RarslaveCommon import * -# Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5) -(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) -config = RarslaveConfig.RarslaveConfig() -logger = RarslaveLogger.RarslaveLogger () +# Global options from the RarslaveGlobals class +options = RarslaveGlobals.options +config = RarslaveGlobals.config +logger = RarslaveGlobals.logger -# Global options to be set / used later. -options = None - -class RarslaveExtractor (object): - - # Instance Variables - # ========================================================================== - # dir -- The directory in which this set lives - # p2files -- All PAR2 files in this set - # name_matched_files -- Files in this set, matched by name only - # prot_matched_files -- Files in this set, matched by parsing PAR2 files only - # type -- This set's type - # heads -- The heads to be extracted - - def __init__ (self, dir, p2files, name_files, prot_files): - - self.dir = dir - self.p2files = p2files - self.name_matched_files = name_files - self.prot_matched_files = prot_files - - # Find the type - self.type = self.__find_type () - - logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug) - - # Find the heads - self.heads = self.__find_heads () - - for h in self.heads: - logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug) - - def __repr__ (self): - return \ - { TYPE_OLDRAR : 'Old RAR', - TYPE_NEWRAR : 'New RAR', - TYPE_ZIP : 'Zip', - TYPE_NOEXTRACT : 'No Extract', - TYPE_UNKNOWN : 'Unknown' } [self.type] - - def __find_type (self): - - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - if self.is_oldrar (all_files): - return TYPE_OLDRAR - elif self.is_newrar (all_files): - return TYPE_NEWRAR - elif self.is_zip (all_files): - return TYPE_ZIP - elif self.is_noextract (all_files): - return TYPE_NOEXTRACT - - return TYPE_UNKNOWN - - def __generic_find_heads (self, regex, ignorecase=True): - - heads = [] - - if ignorecase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - for f in all_files: - if cregex.match (f): - heads.append (f) - - return heads - - def __find_heads (self): - - if self.type == TYPE_OLDRAR: - return self.__generic_find_heads ('^.*\.rar$') - elif self.type == TYPE_NEWRAR: - return self.__generic_find_heads ('^.*\.part0*1\.rar$') - elif self.type == TYPE_ZIP: - return self.__generic_find_heads ('^.*\.zip$') - elif self.type == TYPE_NOEXTRACT: - return self.prot_matched_files - - return [] - - def __create_directory (self, dir): - if dir == None: - return SUCCESS - - if os.path.isdir (dir): - return SUCCESS - - try: - os.makedirs (dir) - logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - return SUCCESS - - def runExtract (self, todir=None): - # Extract all heads of this set - - # Extract to the head's dir if we don't care where to extract - if todir == None: - todir = self.dir - - # Create the directory $todir if it doesn't exist - ret = self.__create_directory (todir) - - if ret != SUCCESS: - return -EEXTRACT - - # Extract all heads - extraction_func = \ - { TYPE_OLDRAR : self.__extract_rar, - TYPE_NEWRAR : self.__extract_rar, - TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract, - TYPE_UNKNOWN : self.__extract_unknown }[self.type] - - # Call the extraction function on each head - for h in self.heads: - full_head = full_abspath (h) - ret = extraction_func (full_head, todir) - logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) - - # Check error code - if ret != SUCCESS: - logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - return SUCCESS - - def __extract_rar (self, file, todir): - assert os.path.isfile (file) - assert os.path.isdir (todir) - - RAR_CMD = config.get_value ('commands', 'unrar') - - cmd = '%s \"%s\"' % (RAR_CMD, file) - ret = run_command (cmd, todir) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_zip (self, file, todir): - ZIP_CMD = config.get_value ('commands', 'unzip') - - cmd = ZIP_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_noextract (self, file, todir): - # Just move this file to the $todir, since no extraction is needed - # FIXME: NOTE: mv will fail by itself if you're moving to the same dir! - NOEXTRACT_CMD = config.get_value ('commands', 'noextract') - - # Make sure that both files are not the same file. If they are, don't run at all. - if os.path.samefile (file, os.path.join (todir, file)): - return SUCCESS - - cmd = NOEXTRACT_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_unknown (self, file, todir): - return SUCCESS - - def __generic_matcher (self, files, regex, nocase=False): - """Run the regex over the files, and see if one matches or not. - NOTE: this does not return the matches, just if a match occurred.""" - - if nocase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - for f in files: - if cregex.match (f): - return True - - return False - - def is_oldrar (self, files): - return self.__generic_matcher (files, '^.*\.r00$') - - def is_newrar (self, files): - return self.__generic_matcher (files, '^.*\.part0*1\.rar$') - - def is_zip (self, files): - return self.__generic_matcher (files, '^.*\.zip$') - - def is_noextract (self, files): - # Type that needs no extraction. - # TODO: Add others ??? - return self.__generic_matcher (files, '^.*\.001$') - -class PAR2Set (object): - - # Instance Variables - # ========================================================================== - # dir -- The directory this set lives in - # p2file -- The starting PAR2 file - # basename -- The basename of the set, guessed from the PAR2 file - # all_p2files -- All PAR2 files of the set, guessed from the PAR2 file name only - # name_matched_files -- Files in this set, guessed by name only - # prot_matched_files -- Files in this set, guessed by parsing the PAR2 only +# A tiny class used to find unique PAR2 sets +class CompareSet (object): def __init__ (self, dir, p2file): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, p2file)) - self.dir = dir self.p2file = p2file - self.basename = self.__get_basename (p2file) - - # Find files that match by name only - self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - # Find all par2 files for this set using name matches - self.all_p2files = find_par2_files (self.name_matched_files) - - # Try to get the protected files for this set - self.prot_matched_files = self.__parse_all_par2 () - - def __list_eq (self, l1, l2): - - if len(l1) != len(l2): - return False - - for e in l1: - if e not in l2: - return False - - return True + self.basename = get_basename (self.p2file) + self.name_matches = find_name_matches (self.dir, self.basename) def __eq__ (self, rhs): - return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ - self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \ - self.__list_eq (self.prot_matched_files, rhs.prot_matched_files) - - def __get_basename (self, name): - """Strips most kinds of endings from a filename""" - - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False - - while not done: - done = True - - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False - - return name - - def __parse_all_par2 (self): - """Searches though self.all_p2files and tries to parse at least one of them""" - done = False - files = [] - - for f in self.all_p2files: - - # Exit early if we've found a good file - if done: - break - - try: - files = Par2Parser.get_protected_files (self.dir, f) - done = True - except (EnvironmentError, OSError, OverflowError): - logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal) - - # Now that we're out of the loop, check if we really finished - if not done: - logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - - # Return whatever we've got, empty or not - return files - - def __find_name_matches (self, dir, basename): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" - - assert os.path.isdir (dir) - - ename = re.escape (basename) - regex = re.compile ('^%s.*$' % (ename, )) - - return [f for f in os.listdir (dir) if regex.match (f)] - - def __update_name_matches (self): - """Updates the self.name_matched_files variable with the most current information. - This should be called after the directory contents are likely to change.""" - - self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - - def __is_joinfile (self, filename): - regex = re.compile ('^.*\.\d\d\d$', re.IGNORECASE) - if regex.match (filename): - return True - - return False - - def __should_be_joined (self, files): - for f in files: - if self.__is_joinfile (f): - return True - - def runCheckAndRepair (self): - PAR2_CMD = config.get_value ('commands', 'par2repair') - - # Get set up - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - join = self.__should_be_joined (all_files) - - # assemble the command - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - command = "%s \"%s\" " % (PAR2_CMD, self.p2file) - - for f in self.all_p2files: - if f != self.p2file: - command += "\"%s\" " % os.path.split (f)[1] - - # Only needed when using par2 to join - if join: - for f in all_files: - if self.__is_joinfile (f): - command += "\"%s\" " % os.path.split (f)[1] - - # run the command - ret = run_command (command, self.dir) + return (self.dir == rhs.dir) \ + and (self.basename == rhs.basename) \ + and list_eq (self.name_matches, rhs.name_matches) - # check the result - if ret != 0: - logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - return SUCCESS - - def __find_deleteable_files (self): - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) - - return [f for f in all_files if dregex.match (f)] - - def __delete_list_of_files (self, dir, files, interactive=False): - # Delete a list of files - - assert os.path.isdir (dir) - - done = False - valid_y = ['Y', 'YES'] - valid_n = ['N', 'NO', ''] - - if interactive: - while not done: - print 'Do you want to delete the following?:' - for f in files: - print f - s = raw_input ('Delete [y/N]: ').upper() - - if s in valid_y + valid_n: - done = True - - if s in valid_n: - return SUCCESS - - for f in files: - try: - os.remove (os.path.join (dir, f)) - logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug) - except: - logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f), - RarslaveLogger.MessageType.Fatal) - return -EDELETE - - return SUCCESS - - def runDelete (self): - deleteable_files = self.__find_deleteable_files () - ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive) - - return ret - - def run_all (self): - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Repair Stage - ret = self.runCheckAndRepair () - - if ret != SUCCESS: - logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - self.__update_name_matches () - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Extraction Stage - extractor = RarslaveExtractor (self.dir, self.all_p2files, \ - self.name_matched_files, self.prot_matched_files) - ret = extractor.runExtract (options.extract_dir) - - if ret != SUCCESS: - logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - self.__update_name_matches () - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Deletion Stage - ret = self.runDelete () - - if ret != SUCCESS: - logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -EDELETE - - logger.addMessage ('Successfully completed: %s' % self.p2file) - return SUCCESS - -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. - - pwd = os.getcwd () - - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) - - ret = os.system (cmd) - os.chdir (pwd) - return ret - -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) - -def find_par2_files (files): - """Find all par2 files in the list $files""" - - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] def find_all_par2_files (dir): """Finds all par2 files in a directory""" @@ -489,10 +43,6 @@ def find_all_par2_files (dir): return find_par2_files (files) -def no_duplicates (li): - """Removes all duplicates from a list""" - return list(set(li)) - def generate_all_parsets (dir): # Generate all parsets in the given directory. @@ -502,11 +52,11 @@ def generate_all_parsets (dir): p2files = find_all_par2_files (dir) for f in p2files: - p = PAR2Set (dir, f) + p = CompareSet (dir, f) if p not in parsets: parsets.append (p) - return parsets + return [(p.dir, p.p2file) for p in parsets] def check_required_progs(): """Check if the required programs are installed""" @@ -610,17 +160,17 @@ def main (): parser = optparse.OptionParser() parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive', - default=config.get_value('options', 'recursive'), + default=config_get_value('options', 'recursive'), help="Don't run recursively") parser.add_option('-d', '--work-dir', dest='work_dir', type='string', - default=config.get_value('directories', 'working_directory'), + default=config_get_value('directories', 'working_directory'), help="Start running at DIR", metavar='DIR') parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string', - default=config.get_value('directories', 'extract_directory'), + default=config_get_value('directories', 'extract_directory'), help="Extract to DIR", metavar='DIR') parser.add_option('-p', '--check-required-programs', @@ -637,7 +187,7 @@ def main (): default=False, help="Write out the current config") parser.add_option('-i', '--interactive', dest='interactive', action='store_true', - default=config.get_value('options', 'interactive'), + default=config_get_value('options', 'interactive'), help="Confirm before removing files") parser.add_option('-q', '--quiet', dest='quiet', action='count', @@ -653,7 +203,8 @@ def main (): # Parse the given options global options - (options, args) = parser.parse_args() + (RarslaveGlobals.options, args) = parser.parse_args() + options = RarslaveGlobals.options # Run any special actions that are needed on these options run_options (options) @@ -665,14 +216,16 @@ def main (): if options.recursive: for (dir, subdirs, files) in os.walk (options.work_dir): parsets = generate_all_parsets (dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Non-recursive else: parsets = generate_all_parsets (options.work_dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Print the results printMessageTable (loglevel)