X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=a8e69bcc38f03d935a821c5463691680b1f30aab;hp=49f72b119ed0cdf1b22f05670f5ce96b8ec768b2;hb=c05c6922b1e23456c194f49b9646615a7a9a75c8;hpb=02e9b17925f6608cde62f6ea3d374ea1c39350c8 diff --git a/rarslave.py b/rarslave.py index 49f72b1..a8e69bc 100644 --- a/rarslave.py +++ b/rarslave.py @@ -10,7 +10,7 @@ import RarslaveConfig import RarslaveLogger # Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) +(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5) (SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) config = RarslaveConfig.RarslaveConfig() logger = RarslaveLogger.RarslaveLogger () @@ -20,45 +20,121 @@ options = None class RarslaveExtractor (object): - def __init__ (self, type): - self.type = type - self.heads = [] + type = None + heads = [] - def addHead (self, dir, head): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, head)) + def __init__ (self, dir, p2files, name_files, prot_files): + + self.dir = dir + self.p2files = p2files + self.name_matched_files = name_files + self.prot_matched_files = prot_files + + # Find the type + self.type = self.__find_type () + + logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug) + + # Find the heads + self.heads = self.__find_heads () + + for h in self.heads: + logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug) + + def __repr__ (self): + return \ + { TYPE_OLDRAR : 'Old RAR', + TYPE_NEWRAR : 'New RAR', + TYPE_ZIP : 'Zip', + TYPE_NOEXTRACT : 'No Extract', + TYPE_UNKNOWN : 'Unknown' } [self.type] + + def __find_type (self): + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + if self.is_oldrar (all_files): + return TYPE_OLDRAR + elif self.is_newrar (all_files): + return TYPE_NEWRAR + elif self.is_zip (all_files): + return TYPE_ZIP + elif self.is_noextract (all_files): + return TYPE_NOEXTRACT - full_head = os.path.join (dir, head) - logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug) - self.heads.append (full_head) + return TYPE_UNKNOWN - def extract (self, todir=None): + def __generic_find_heads (self, regex, ignorecase=True): + + heads = [] + + if ignorecase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + for f in all_files: + if cregex.match (f): + heads.append (f) + + return heads + + def __find_heads (self): + + if self.type == TYPE_OLDRAR: + return self.__generic_find_heads ('^.*\.rar$') + elif self.type == TYPE_NEWRAR: + return self.__generic_find_heads ('^.*\.part0*1\.rar$') + elif self.type == TYPE_ZIP: + return self.__generic_find_heads ('^.*\.zip$') + elif self.type == TYPE_NOEXTRACT: + return self.prot_matched_files + + return [] + + def __create_directory (self, dir): + if dir == None: + return SUCCESS + + if os.path.isdir (dir): + return SUCCESS + + try: + os.makedirs (dir) + logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose) + except OSError: + logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + return SUCCESS + + def runExtract (self, todir=None): # Extract all heads of this set + # Extract to the head's dir if we don't care where to extract + if todir == None: + todir = self.dir + # Create the directory $todir if it doesn't exist - if todir != None and not os.path.isdir (todir): - logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose) - try: - os.makedirs (todir) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT + ret = self.__create_directory (todir) + + if ret != SUCCESS: + return -EEXTRACT # Extract all heads extraction_func = \ { TYPE_OLDRAR : self.__extract_rar, TYPE_NEWRAR : self.__extract_rar, TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + TYPE_NOEXTRACT : self.__extract_noextract, + TYPE_UNKNOWN : self.__extract_unknown }[self.type] # Call the extraction function on each head for h in self.heads: - if todir == None: - # Run in the head's directory - ret = extraction_func (h, os.path.dirname (h)) - else: - ret = extraction_func (h, todir) - + full_head = full_abspath (h) + ret = extraction_func (full_head, todir) logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) # Check error code @@ -113,331 +189,302 @@ class RarslaveExtractor (object): return SUCCESS + def __extract_unknown (self, file, todir): + return SUCCESS + def __generic_matcher (self, files, regex, nocase=False): + """Run the regex over the files, and see if one matches or not. + NOTE: this does not return the matches, just if a match occurred.""" -class RarslaveRepairer (object): - # Verify (and repair) the set - # Make sure it worked, otherwise clean up and return failure - - def __init__ (self, dir, file, join=False): - self.dir = dir # the directory containing the par2 file - self.file = file # the par2 file - self.join = join # True if the par2 set is 001 002 ... + if nocase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) + for f in files: + if cregex.match (f): + return True - def checkAndRepair (self): - # Form the command: - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - PAR2_CMD = config.get_value ('commands', 'par2repair') + return False - # Get set up - basename = get_basename (self.file) - all_files = find_likely_files (self.dir, self.file) - all_files.sort () - par2_files = find_par2_files (all_files) + def is_oldrar (self, files): + return self.__generic_matcher (files, '^.*\.r00$') - # assemble the command - command = "%s \"%s\" " % (PAR2_CMD, self.file) + def is_newrar (self, files): + return self.__generic_matcher (files, '^.*\.part0*1\.rar$') - for f in par2_files: - if f != self.file: - command += "\"%s\" " % os.path.split (f)[1] + def is_zip (self, files): + return self.__generic_matcher (files, '^.*\.zip$') - if self.join: - for f in all_files: - if f not in par2_files: - command += "\"%s\" " % os.path.split (f)[1] + def is_noextract (self, files): + # Type that needs no extraction. + # TODO: Add others ??? + return self.__generic_matcher (files, '^.*\.001$') - # run the command - ret = run_command (command, self.dir) +class PAR2Set (object): - # check the result - if ret != 0: - logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal) - return -ECHECK + dir = None + p2file = None # The starting par2 + basename = None # The p2file's basename + all_p2files = [] + name_matched_files = [] # Files that match by basename of the p2file + prot_matched_files = [] # Files that match by being protected members - return SUCCESS + def __init__ (self, dir, p2file): + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, p2file)) -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. + self.dir = dir + self.p2file = p2file + self.basename = self.__get_basename (p2file) - pwd = os.getcwd () + # Find files that match by name only + self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) + # Find all par2 files for this set using name matches + self.all_p2files = find_par2_files (self.name_matched_files) - ret = os.system (cmd) - os.chdir (pwd) - return ret + # Try to get the protected files for this set + self.prot_matched_files = self.__parse_all_par2 () -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) + def __list_eq (self, l1, l2): -def get_basename (name): - """Strips most kinds of endings from a filename""" + if len(l1) != len(l2): + return False - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False + for e in l1: + if e not in l2: + return False - while not done: - done = True + return True - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False + def __eq__ (self, rhs): + return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ + self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \ + self.__list_eq (self.prot_matched_files, rhs.prot_matched_files) - return name + def __get_basename (self, name): + """Strips most kinds of endings from a filename""" -def find_likely_files (dir, p2file): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" + regex = config.get_value ('regular expressions', 'basename_regex') + r = re.compile (regex, re.IGNORECASE) + done = False - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, p2file)) + while not done: + done = True - basename = get_basename (p2file) + if r.match (name): + g = r.match (name).groups() + name = g[0] + done = False - dir = os.path.abspath (dir) - ename = re.escape (basename) - regex = re.compile ('^%s.*$' % (ename, )) + return name - name_matches = [f for f in os.listdir (dir) if regex.match (f)] - try: - parsed_matches = Par2Parser.get_protected_files (dir, p2file) - except EnvironmentError: - parsed_matches = [] - logger.addMessage ('Bad par2 file: %s' % p2file, RarslaveLogger.MessageType.Fatal) + def __parse_all_par2 (self): + """Searches though self.all_p2files and tries to parse at least one of them""" + done = False + files = [] - return name_matches + parsed_matches + for f in self.all_p2files: -def find_par2_files (files): - """Find all par2 files in the list $files""" + # Exit early if we've found a good file + if done: + break - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] + try: + files = Par2Parser.get_protected_files (self.dir, f) + done = True + except (EnvironmentError, OSError, OverflowError): + logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal) -def find_all_par2_files (dir): - """Finds all par2 files in a directory""" - # NOTE: does NOT return absolute paths + # Now that we're out of the loop, check if we really finished + if not done: + logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given + # Return whatever we've got, empty or not + return files - dir = os.path.abspath (dir) - files = os.listdir (dir) + def __find_name_matches (self, dir, basename): + """Finds files which are likely to be part of the set corresponding + to $name in the directory $dir""" - return find_par2_files (files) + assert os.path.isdir (dir) -def find_extraction_heads (dir, files): - """Takes a list of possible files and finds likely heads of - extraction.""" + ename = re.escape (basename) + regex = re.compile ('^%s.*$' % (ename, )) - # NOTE: perhaps this should happen AFTER repair is - # NOTE: successful. That way all files would already exist + return [f for f in os.listdir (dir) if regex.match (f)] - # According to various sources online: - # 1) pre rar-3.0: .rar .r00 .r01 ... - # 2) post rar-3.0: .part01.rar .part02.rar - # 3) zip all ver: .zip + def __update_name_matches (self): + """Updates the self.name_matched_files variable with the most current information. + This should be called after the directory contents are likely to change.""" - extractor = None - p2files = find_par2_files (files) + self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - # Old RAR type, find all files ending in .rar - if is_oldrar (files): - extractor = RarslaveExtractor (TYPE_OLDRAR) - regex = re.compile ('^.*\.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) + def __is_joinfile (self, filename): + regex = re.compile ('^.*\.\d\d\d$', re.IGNORECASE) + if regex.match (filename): + return True - if is_newrar (files): - extractor = RarslaveExtractor (TYPE_NEWRAR) - regex = re.compile ('^.*\.part0*1.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) + return False - if is_zip (files): - extractor = RarslaveExtractor (TYPE_ZIP) - regex = re.compile ('^.*\.zip$', re.IGNORECASE) + def __should_be_joined (self, files): for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_noextract (files): - # Use the Par2 Parser (from cfv) here to find out what files are protected. - # Since these are not being extracted, they will be mv'd to another directory - # later. - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - for f in p2files: - done = False - try: - prot_files = Par2Parser.get_protected_files (dir, f) - done = True - except EnvironmentError: - logger.addMessage ('Error parsing PAR2 file: %s', f) - continue - - if done: - break - - if done: - for f in prot_files: - extractor.addHead (dir, f) - else: - logger.addMessage ('Error parsing all PAR2 files in this set ...') - - # Make sure we found the type - if extractor == None: - logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0], - RarslaveLogger.MessageType.Verbose) - - # No-heads here, but it's better than failing completely - extractor = RarslaveExtractor (TYPE_NOEXTRACT) + if self.__is_joinfile (f): + return True - return extractor - -def generic_matcher (files, regex, nocase=False): - """Run the regex over the files, and see if one matches or not. - NOTE: this does not return the matches, just if a match occurred.""" - - if nocase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - for f in files: - if cregex.match (f): - return True + def runCheckAndRepair (self): + PAR2_CMD = config.get_value ('commands', 'par2repair') - return False + # Get set up + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + join = self.__should_be_joined (all_files) -def is_oldrar (files): - return generic_matcher (files, '^.*\.r00$') + # assemble the command + # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] + command = "%s \"%s\" " % (PAR2_CMD, self.p2file) -def is_newrar (files): - return generic_matcher (files, '^.*\.part0*1\.rar$') + for f in self.all_p2files: + if f != self.p2file: + command += "\"%s\" " % os.path.split (f)[1] -def is_zip (files): - return generic_matcher (files, '^.*\.zip$') + # Only needed when using par2 to join + if join: + for f in all_files: + if self.__is_joinfile (f): + command += "\"%s\" " % os.path.split (f)[1] -def is_noextract (files): - # Type that needs no extraction. - # TODO: Add others ??? - return generic_matcher (files, '^.*\.001$') + # run the command + ret = run_command (command, self.dir) -def find_deleteable_files (files): - # Deleteable types regex should come from the config - dfiles = [] - DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + # check the result + if ret != 0: + logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) + return -ECHECK - return [f for f in files if dregex.match (f)] + return SUCCESS -def printlist (li): - for f in li: - print f + def __find_deleteable_files (self): + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') + dregex = re.compile (DELETE_REGEX, re.IGNORECASE) -class PAR2Set (object): + return [f for f in all_files if dregex.match (f)] - dir = None - file = None - likely_files = [] + def __delete_list_of_files (self, dir, files, interactive=False): + # Delete a list of files - def __init__ (self, dir, file): assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) - self.dir = dir - self.file = file + done = False + valid_y = ['Y', 'YES'] + valid_n = ['N', 'NO', ''] - basename = get_basename (file) - self.likely_files = find_likely_files (dir, file) + if interactive: + while not done: + print 'Do you want to delete the following?:' + for f in files: + print f + s = raw_input ('Delete [y/N]: ').upper() - def __list_eq (self, l1, l2): + if s in valid_y + valid_n: + done = True - if len(l1) != len(l2): - return False + if s in valid_n: + return SUCCESS - for e in l1: - if e not in l2: - return False + for f in files: + try: + os.remove (os.path.join (dir, f)) + logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug) + except: + logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f), + RarslaveLogger.MessageType.Fatal) + return -EDELETE - return True + return SUCCESS - def __eq__ (self, rhs): - return self.__list_eq (self.likely_files, rhs.likely_files) + def runDelete (self): + deleteable_files = self.__find_deleteable_files () + ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive) - def run_all (self): - par2files = find_par2_files (self.likely_files) - par2head = par2files[0] + return ret - join = is_noextract (self.likely_files) + def run_all (self): + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) # Repair Stage - repairer = RarslaveRepairer (self.dir, par2head, join) - ret = repairer.checkAndRepair () + ret = self.runCheckAndRepair () if ret != SUCCESS: - logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -ECHECK + self.__update_name_matches () + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + # Extraction Stage - EXTRACT_DIR = options.extract_dir - extractor = find_extraction_heads (self.dir, self.likely_files) - ret = extractor.extract (EXTRACT_DIR) + extractor = RarslaveExtractor (self.dir, self.all_p2files, \ + self.name_matched_files, self.prot_matched_files) + ret = extractor.runExtract (options.extract_dir) if ret != SUCCESS: - logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -EEXTRACT + self.__update_name_matches () + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + # Deletion Stage - DELETE_INTERACTIVE = options.interactive - deleteable_files = find_deleteable_files (self.likely_files) - ret = delete_list (self.dir, deleteable_files, DELETE_INTERACTIVE) + ret = self.runDelete () if ret != SUCCESS: - logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -EDELETE - logger.addMessage ('Successfully completed: %s' % par2head) + logger.addMessage ('Successfully completed: %s' % self.p2file) return SUCCESS -def delete_list (dir, files, interactive=False): - # Delete a list of files +def run_command (cmd, indir=None): + # Runs the specified command-line in the directory given (or, in the current directory + # if none is given). It returns the status code given by the application. - assert os.path.isdir (dir) + pwd = os.getcwd () - done = False - valid_y = ['Y', 'YES'] - valid_n = ['N', 'NO'] + if indir != None: + assert os.path.isdir (indir) # MUST be a directory! + os.chdir (indir) - if interactive: - while not done: - print 'Do you want to delete the following?:' - printlist (files) - s = raw_input ('Delete [y/N]: ').upper() + print 'RUNNING (%s): %s' % (indir, cmd) + ret = os.system (cmd) + os.chdir (pwd) + return ret - if s in valid_y + valid_n: - done = True +def full_abspath (p): + return os.path.abspath (os.path.expanduser (p)) - if s in valid_n: - return SUCCESS +def find_par2_files (files): + """Find all par2 files in the list $files""" + + PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') + regex = re.compile (PAR2_REGEX, re.IGNORECASE) + return [f for f in files if regex.match (f)] - for f in files: - os.remove (os.path.join (dir, f)) +def find_all_par2_files (dir): + """Finds all par2 files in a directory""" + # NOTE: does NOT return absolute paths - return SUCCESS + if not os.path.isdir (os.path.abspath (dir)): + raise ValueError # bad directory given + + dir = os.path.abspath (dir) + files = os.listdir (dir) + + return find_par2_files (files) +def no_duplicates (li): + """Removes all duplicates from a list""" + return list(set(li)) def generate_all_parsets (dir): # Generate all parsets in the given directory.