X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=a8e69bcc38f03d935a821c5463691680b1f30aab;hp=eb133a1881b81df204c6f8f96abe124ec26b787a;hb=c05c6922b1e23456c194f49b9646615a7a9a75c8;hpb=2c789e235b0fe20478427e49148b9697f182ac95 diff --git a/rarslave.py b/rarslave.py index eb133a1..a8e69bc 100644 --- a/rarslave.py +++ b/rarslave.py @@ -5,12 +5,12 @@ VERSION="2.0.0" PROGRAM="rarslave2" import re, os, sys, optparse -import par2parser +import Par2Parser import RarslaveConfig import RarslaveLogger # Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) +(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5) (SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) config = RarslaveConfig.RarslaveConfig() logger = RarslaveLogger.RarslaveLogger () @@ -20,45 +20,121 @@ options = None class RarslaveExtractor (object): - def __init__ (self, type): - self.type = type - self.heads = [] + type = None + heads = [] - def addHead (self, dir, head): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, head)) + def __init__ (self, dir, p2files, name_files, prot_files): + + self.dir = dir + self.p2files = p2files + self.name_matched_files = name_files + self.prot_matched_files = prot_files + + # Find the type + self.type = self.__find_type () + + logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug) + + # Find the heads + self.heads = self.__find_heads () + + for h in self.heads: + logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug) + + def __repr__ (self): + return \ + { TYPE_OLDRAR : 'Old RAR', + TYPE_NEWRAR : 'New RAR', + TYPE_ZIP : 'Zip', + TYPE_NOEXTRACT : 'No Extract', + TYPE_UNKNOWN : 'Unknown' } [self.type] + + def __find_type (self): + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + if self.is_oldrar (all_files): + return TYPE_OLDRAR + elif self.is_newrar (all_files): + return TYPE_NEWRAR + elif self.is_zip (all_files): + return TYPE_ZIP + elif self.is_noextract (all_files): + return TYPE_NOEXTRACT - full_head = os.path.join (dir, head) - logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug) - self.heads.append (full_head) + return TYPE_UNKNOWN + + def __generic_find_heads (self, regex, ignorecase=True): + + heads = [] + + if ignorecase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + for f in all_files: + if cregex.match (f): + heads.append (f) + + return heads + + def __find_heads (self): + + if self.type == TYPE_OLDRAR: + return self.__generic_find_heads ('^.*\.rar$') + elif self.type == TYPE_NEWRAR: + return self.__generic_find_heads ('^.*\.part0*1\.rar$') + elif self.type == TYPE_ZIP: + return self.__generic_find_heads ('^.*\.zip$') + elif self.type == TYPE_NOEXTRACT: + return self.prot_matched_files + + return [] + + def __create_directory (self, dir): + if dir == None: + return SUCCESS - def extract (self, todir=None): + if os.path.isdir (dir): + return SUCCESS + + try: + os.makedirs (dir) + logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose) + except OSError: + logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + return SUCCESS + + def runExtract (self, todir=None): # Extract all heads of this set + # Extract to the head's dir if we don't care where to extract + if todir == None: + todir = self.dir + # Create the directory $todir if it doesn't exist - if todir != None and not os.path.isdir (todir): - logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose) - try: - os.makedirs (todir) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT + ret = self.__create_directory (todir) + + if ret != SUCCESS: + return -EEXTRACT # Extract all heads extraction_func = \ { TYPE_OLDRAR : self.__extract_rar, TYPE_NEWRAR : self.__extract_rar, TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + TYPE_NOEXTRACT : self.__extract_noextract, + TYPE_UNKNOWN : self.__extract_unknown }[self.type] # Call the extraction function on each head for h in self.heads: - if todir == None: - # Run in the head's directory - ret = extraction_func (h, os.path.dirname (h)) - else: - ret = extraction_func (h, todir) - + full_head = full_abspath (h) + ret = extraction_func (full_head, todir) logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) # Check error code @@ -113,332 +189,302 @@ class RarslaveExtractor (object): return SUCCESS - - -class RarslaveRepairer (object): - # Verify (and repair) the set - # Make sure it worked, otherwise clean up and return failure - - def __init__ (self, dir, file, join=False): - self.dir = dir # the directory containing the par2 file - self.file = file # the par2 file - self.join = join # True if the par2 set is 001 002 ... - - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) - - def checkAndRepair (self): - # Form the command: - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - PAR2_CMD = config.get_value ('commands', 'par2repair') - - # Get set up - basename = get_basename (self.file) - all_files = find_likely_files (basename, self.dir) - all_files.sort () - par2_files = find_par2_files (all_files) - - # assemble the command - command = "%s \"%s\" " % (PAR2_CMD, self.file) - - for f in par2_files: - if f != self.file: - command += "\"%s\" " % os.path.split (f)[1] - - if self.join: - for f in all_files: - if f not in par2_files: - command += "\"%s\" " % os.path.split (f)[1] - - # run the command - ret = run_command (command, self.dir) - - # check the result - if ret != 0: - logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - + def __extract_unknown (self, file, todir): return SUCCESS -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. + def __generic_matcher (self, files, regex, nocase=False): + """Run the regex over the files, and see if one matches or not. + NOTE: this does not return the matches, just if a match occurred.""" - pwd = os.getcwd () + if nocase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) + for f in files: + if cregex.match (f): + return True - ret = os.system (cmd) - os.chdir (pwd) - return ret + return False -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) + def is_oldrar (self, files): + return self.__generic_matcher (files, '^.*\.r00$') -def get_basename (name): - """Strips most kinds of endings from a filename""" + def is_newrar (self, files): + return self.__generic_matcher (files, '^.*\.part0*1\.rar$') - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False + def is_zip (self, files): + return self.__generic_matcher (files, '^.*\.zip$') - while not done: - done = True + def is_noextract (self, files): + # Type that needs no extraction. + # TODO: Add others ??? + return self.__generic_matcher (files, '^.*\.001$') - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False +class PAR2Set (object): - return name + dir = None + p2file = None # The starting par2 + basename = None # The p2file's basename + all_p2files = [] + name_matched_files = [] # Files that match by basename of the p2file + prot_matched_files = [] # Files that match by being protected members -def find_likely_files (name, dir): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" + def __init__ (self, dir, p2file): + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, p2file)) - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given + self.dir = dir + self.p2file = p2file + self.basename = self.__get_basename (p2file) - dir = os.path.abspath (dir) - ename = re.escape (name) - regex = re.compile ('^%s.*$' % (ename, )) + # Find files that match by name only + self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - return [f for f in os.listdir (dir) if regex.match (f)] + # Find all par2 files for this set using name matches + self.all_p2files = find_par2_files (self.name_matched_files) -def find_par2_files (files): - """Find all par2 files in the list $files""" + # Try to get the protected files for this set + self.prot_matched_files = self.__parse_all_par2 () - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] + def __list_eq (self, l1, l2): -def find_all_par2_files (dir): - """Finds all par2 files in a directory""" - # NOTE: does NOT return absolute paths + if len(l1) != len(l2): + return False - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given + for e in l1: + if e not in l2: + return False - dir = os.path.abspath (dir) - files = os.listdir (dir) + return True - return find_par2_files (files) + def __eq__ (self, rhs): + return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ + self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \ + self.__list_eq (self.prot_matched_files, rhs.prot_matched_files) -def has_extension (f, ext): - """Checks if f has the extension ext""" + def __get_basename (self, name): + """Strips most kinds of endings from a filename""" - if ext[0] != '.': - ext = '.' + ext + regex = config.get_value ('regular expressions', 'basename_regex') + r = re.compile (regex, re.IGNORECASE) + done = False - ext = re.escape (ext) - regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE) - return regex.match (f) + while not done: + done = True -def find_extraction_heads (dir, files): - """Takes a list of possible files and finds likely heads of - extraction.""" + if r.match (name): + g = r.match (name).groups() + name = g[0] + done = False - # NOTE: perhaps this should happen AFTER repair is - # NOTE: successful. That way all files would already exist + return name - # According to various sources online: - # 1) pre rar-3.0: .rar .r00 .r01 ... - # 2) post rar-3.0: .part01.rar .part02.rar - # 3) zip all ver: .zip + def __parse_all_par2 (self): + """Searches though self.all_p2files and tries to parse at least one of them""" + done = False + files = [] - extractor = None - p2files = find_par2_files (files) + for f in self.all_p2files: - # Old RAR type, find all files ending in .rar - if is_oldrar (files): - extractor = RarslaveExtractor (TYPE_OLDRAR) - regex = re.compile ('^.*\.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) + # Exit early if we've found a good file + if done: + break - if is_newrar (files): - extractor = RarslaveExtractor (TYPE_NEWRAR) - regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) + try: + files = Par2Parser.get_protected_files (self.dir, f) + done = True + except (EnvironmentError, OSError, OverflowError): + logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal) - if is_zip (files): - extractor = RarslaveExtractor (TYPE_ZIP) - regex = re.compile ('^.*\.zip$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) + # Now that we're out of the loop, check if we really finished + if not done: + logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - if is_noextract (files): - # Use the Par2 Parser (from cfv) here to find out what files are protected. - # Since these are not being extracted, they will be mv'd to another directory - # later. - extractor = RarslaveExtractor (TYPE_NOEXTRACT) + # Return whatever we've got, empty or not + return files - for f in p2files: - done = False - try: - prot_files = par2parser.get_protected_files (dir, f) - done = True - except: #FIXME: add the actual exceptions - logger.addMessage ('Error parsing PAR2 file: %s', f) - continue + def __find_name_matches (self, dir, basename): + """Finds files which are likely to be part of the set corresponding + to $name in the directory $dir""" - if done: - break + assert os.path.isdir (dir) - if done: - for f in prot_files: - extractor.addHead (dir, f) - else: - logger.addMessage ('Error parsing all PAR2 files in this set ...') + ename = re.escape (basename) + regex = re.compile ('^%s.*$' % (ename, )) - # Make sure we found the type - if extractor == None: - logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0], - RarslaveLogger.MessageType.Fatal) + return [f for f in os.listdir (dir) if regex.match (f)] - # No-heads here, but it's better than failing completely - extractor = RarslaveExtractor (TYPE_NOEXTRACT) + def __update_name_matches (self): + """Updates the self.name_matched_files variable with the most current information. + This should be called after the directory contents are likely to change.""" - return extractor + self.name_matched_files = self.__find_name_matches (self.dir, self.basename) -def is_oldrar (files): - for f in files: - if has_extension (f, '.r00'): + def __is_joinfile (self, filename): + regex = re.compile ('^.*\.\d\d\d$', re.IGNORECASE) + if regex.match (filename): return True - return False + return False -def is_newrar (files): - for f in files: - if has_extension (f, '.part01.rar'): - return True + def __should_be_joined (self, files): + for f in files: + if self.__is_joinfile (f): + return True - return False + def runCheckAndRepair (self): + PAR2_CMD = config.get_value ('commands', 'par2repair') -def is_zip (files): - for f in files: - if has_extension (f, '.zip'): - return True + # Get set up + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + join = self.__should_be_joined (all_files) - return False + # assemble the command + # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] + command = "%s \"%s\" " % (PAR2_CMD, self.p2file) -def is_noextract (files): - # Type that needs no extraction. - # TODO: Add others ??? - for f in files: - if has_extension (f, '.001'): - return True + for f in self.all_p2files: + if f != self.p2file: + command += "\"%s\" " % os.path.split (f)[1] - return False + # Only needed when using par2 to join + if join: + for f in all_files: + if self.__is_joinfile (f): + command += "\"%s\" " % os.path.split (f)[1] -def find_deleteable_files (files): - # Deleteable types regex should come from the config - dfiles = [] - DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + # run the command + ret = run_command (command, self.dir) - return [f for f in files if dregex.match (f)] + # check the result + if ret != 0: + logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) + return -ECHECK -def printlist (li): - for f in li: - print f + return SUCCESS -class PAR2Set (object): + def __find_deleteable_files (self): + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') + dregex = re.compile (DELETE_REGEX, re.IGNORECASE) - dir = None - file = None - likely_files = [] + return [f for f in all_files if dregex.match (f)] + + def __delete_list_of_files (self, dir, files, interactive=False): + # Delete a list of files - def __init__ (self, dir, file): assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) - self.dir = dir - self.file = file + done = False + valid_y = ['Y', 'YES'] + valid_n = ['N', 'NO', ''] - basename = get_basename (file) - self.likely_files = find_likely_files (basename, dir) + if interactive: + while not done: + print 'Do you want to delete the following?:' + for f in files: + print f + s = raw_input ('Delete [y/N]: ').upper() - def __list_eq (self, l1, l2): + if s in valid_y + valid_n: + done = True - if len(l1) != len(l2): - return False + if s in valid_n: + return SUCCESS - for e in l1: - if e not in l2: - return False + for f in files: + try: + os.remove (os.path.join (dir, f)) + logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug) + except: + logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f), + RarslaveLogger.MessageType.Fatal) + return -EDELETE - return True + return SUCCESS - def __eq__ (self, rhs): - return self.__list_eq (self.likely_files, rhs.likely_files) + def runDelete (self): + deleteable_files = self.__find_deleteable_files () + ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive) - def run_all (self): - par2files = find_par2_files (self.likely_files) - par2head = par2files[0] + return ret - join = is_noextract (self.likely_files) + def run_all (self): + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) # Repair Stage - repairer = RarslaveRepairer (self.dir, par2head, join) - ret = repairer.checkAndRepair () + ret = self.runCheckAndRepair () if ret != SUCCESS: - logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -ECHECK + self.__update_name_matches () + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + # Extraction Stage - EXTRACT_DIR = options.extract_dir - extractor = find_extraction_heads (self.dir, self.likely_files) - ret = extractor.extract (EXTRACT_DIR) + extractor = RarslaveExtractor (self.dir, self.all_p2files, \ + self.name_matched_files, self.prot_matched_files) + ret = extractor.runExtract (options.extract_dir) if ret != SUCCESS: - logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -EEXTRACT + self.__update_name_matches () + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + # Deletion Stage - DELETE_INTERACTIVE = options.interactive - deleteable_files = find_deleteable_files (self.likely_files) - ret = delete_list (self.dir, deleteable_files, DELETE_INTERACTIVE) + ret = self.runDelete () if ret != SUCCESS: - logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -EDELETE - logger.addMessage ('Successfully completed: %s' % par2head) + logger.addMessage ('Successfully completed: %s' % self.p2file) return SUCCESS -def delete_list (dir, files, interactive=False): - # Delete a list of files +def run_command (cmd, indir=None): + # Runs the specified command-line in the directory given (or, in the current directory + # if none is given). It returns the status code given by the application. - assert os.path.isdir (dir) + pwd = os.getcwd () - done = False - valid_y = ['Y', 'YES'] - valid_n = ['N', 'NO'] + if indir != None: + assert os.path.isdir (indir) # MUST be a directory! + os.chdir (indir) - if interactive: - while not done: - print 'Do you want to delete the following?:' - s = raw_input ('Delete [y/N]: ').upper() + print 'RUNNING (%s): %s' % (indir, cmd) + ret = os.system (cmd) + os.chdir (pwd) + return ret - if s in valid_y + valid_n: - done = True +def full_abspath (p): + return os.path.abspath (os.path.expanduser (p)) - if s in valid_n: - return SUCCESS +def find_par2_files (files): + """Find all par2 files in the list $files""" - for f in files: - os.remove (os.path.join (dir, f)) + PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') + regex = re.compile (PAR2_REGEX, re.IGNORECASE) + return [f for f in files if regex.match (f)] - return SUCCESS +def find_all_par2_files (dir): + """Finds all par2 files in a directory""" + # NOTE: does NOT return absolute paths + + if not os.path.isdir (os.path.abspath (dir)): + raise ValueError # bad directory given + + dir = os.path.abspath (dir) + files = os.listdir (dir) + + return find_par2_files (files) +def no_duplicates (li): + """Removes all duplicates from a list""" + return list(set(li)) def generate_all_parsets (dir): # Generate all parsets in the given directory.