From 17de3f5e52714bcdad168416ea14b2aebb5db887 Mon Sep 17 00:00:00 2001 From: "Ira W. Snyder" Date: Thu, 28 Dec 2006 16:56:18 -0800 Subject: [PATCH] [RARSLAVE] Major Refactoring This continues the refactoring started in the last commit. The changes encompass both the PAR2Set and RarslaveExtractor classes. Signed-off-by: Ira W. Snyder --- Makefile | 2 + rarslave.py | 384 +++++++++++++++++++++++++++------------------------- 2 files changed, 200 insertions(+), 186 deletions(-) diff --git a/Makefile b/Makefile index 8590fff..7fd2ad2 100644 --- a/Makefile +++ b/Makefile @@ -10,3 +10,5 @@ test: run: clean python rarslave.py + +.PHONY: all clean test run diff --git a/rarslave.py b/rarslave.py index ee8be7e..6972d41 100644 --- a/rarslave.py +++ b/rarslave.py @@ -10,7 +10,7 @@ import RarslaveConfig import RarslaveLogger # Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) +(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5) (SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) config = RarslaveConfig.RarslaveConfig() logger = RarslaveLogger.RarslaveLogger () @@ -20,45 +20,121 @@ options = None class RarslaveExtractor (object): - def __init__ (self, type): - self.type = type - self.heads = [] + type = None + heads = [] - def addHead (self, dir, head): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, head)) + def __init__ (self, dir, p2files, name_files, prot_files): + + self.dir = dir + self.p2files = p2files + self.name_matched_files = name_files + self.prot_matched_files = prot_files + + # Find the type + self.type = self.__find_type () + + logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug) + + # Find the heads + self.heads = self.__find_heads () + + for h in self.heads: + logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug) + + def __repr__ (self): + return \ + { TYPE_OLDRAR : 'Old RAR', + TYPE_NEWRAR : 'New RAR', + TYPE_ZIP : 'Zip', + TYPE_NOEXTRACT : 'No Extract', + TYPE_UNKNOWN : 'Unknown' } [self.type] + + def __find_type (self): + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + if self.is_oldrar (all_files): + return TYPE_OLDRAR + elif self.is_newrar (all_files): + return TYPE_NEWRAR + elif self.is_zip (all_files): + return TYPE_ZIP + elif self.is_noextract (all_files): + return TYPE_NOEXTRACT + + return TYPE_UNKNOWN + + def __generic_find_heads (self, regex, ignorecase=True): + + heads = [] + + if ignorecase: + cregex = re.compile (regex, re.IGNORECASE) + else: + cregex = re.compile (regex) + + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + + for f in all_files: + if cregex.match (f): + heads.append (f) - full_head = os.path.join (dir, head) - logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug) - self.heads.append (full_head) + return heads - def extract (self, todir=None): + def __find_heads (self): + + if self.type == TYPE_OLDRAR: + return self.__generic_find_heads ('^.*\.rar$') + elif self.type == TYPE_NEWRAR: + return self.__generic_find_heads ('^.*\.part0*1\.rar$') + elif self.type == TYPE_ZIP: + return self.__generic_find_heads ('^.*\.zip$') + elif self.type == TYPE_NOEXTRACT: + return self.prot_matched_files + + return [] + + def __create_directory (self, dir): + if dir == None: + return SUCCESS + + if os.path.isdir (dir): + return SUCCESS + + try: + os.makedirs (dir) + logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose) + except OSError: + logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + return SUCCESS + + def runExtract (self, todir=None): # Extract all heads of this set + # Extract to the head's dir if we don't care where to extract + if todir == None: + todir = self.dir + # Create the directory $todir if it doesn't exist - if todir != None and not os.path.isdir (todir): - logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose) - try: - os.makedirs (todir) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT + ret = self.__create_directory (todir) + + if ret != SUCCESS: + return -EEXTRACT # Extract all heads extraction_func = \ { TYPE_OLDRAR : self.__extract_rar, TYPE_NEWRAR : self.__extract_rar, TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + TYPE_NOEXTRACT : self.__extract_noextract, + TYPE_UNKNOWN : self.__extract_unknown }[self.type] # Call the extraction function on each head for h in self.heads: - if todir == None: - # Run in the head's directory - ret = extraction_func (h, os.path.dirname (h)) - else: - ret = extraction_func (h, todir) - + full_head = full_abspath (h) + ret = extraction_func (full_head, todir) logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) # Check error code @@ -113,162 +189,37 @@ class RarslaveExtractor (object): return SUCCESS -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. - - pwd = os.getcwd () - - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) - - ret = os.system (cmd) - os.chdir (pwd) - return ret - -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) - -def get_basename (name): - """Strips most kinds of endings from a filename""" - - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False - - while not done: - done = True - - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False - - return name - -def find_par2_files (files): - """Find all par2 files in the list $files""" - - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] - -def find_all_par2_files (dir): - """Finds all par2 files in a directory""" - # NOTE: does NOT return absolute paths - - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given - - dir = os.path.abspath (dir) - files = os.listdir (dir) - - return find_par2_files (files) - -def find_extraction_heads (dir, files): - """Takes a list of possible files and finds likely heads of - extraction.""" - - # NOTE: perhaps this should happen AFTER repair is - # NOTE: successful. That way all files would already exist - - # According to various sources online: - # 1) pre rar-3.0: .rar .r00 .r01 ... - # 2) post rar-3.0: .part01.rar .part02.rar - # 3) zip all ver: .zip - - extractor = None - p2files = find_par2_files (files) - - # Old RAR type, find all files ending in .rar - if is_oldrar (files): - extractor = RarslaveExtractor (TYPE_OLDRAR) - regex = re.compile ('^.*\.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_newrar (files): - extractor = RarslaveExtractor (TYPE_NEWRAR) - regex = re.compile ('^.*\.part0*1.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_zip (files): - extractor = RarslaveExtractor (TYPE_ZIP) - regex = re.compile ('^.*\.zip$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_noextract (files): - # Use the Par2 Parser (from cfv) here to find out what files are protected. - # Since these are not being extracted, they will be mv'd to another directory - # later. - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - for f in p2files: - done = False - try: - prot_files = Par2Parser.get_protected_files (dir, f) - done = True - except (EnvironmentError, OverflowError, OSError): - logger.addMessage ('Error parsing PAR2 file: %s', f) - continue + def __extract_unknown (self, file, todir): + return SUCCESS - if done: - break + def __generic_matcher (self, files, regex, nocase=False): + """Run the regex over the files, and see if one matches or not. + NOTE: this does not return the matches, just if a match occurred.""" - if done: - for f in prot_files: - extractor.addHead (dir, f) + if nocase: + cregex = re.compile (regex, re.IGNORECASE) else: - logger.addMessage ('Error parsing all PAR2 files in this set ...') - - # Make sure we found the type - if extractor == None: - logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0], - RarslaveLogger.MessageType.Verbose) - - # No-heads here, but it's better than failing completely - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - return extractor - -def generic_matcher (files, regex, nocase=False): - """Run the regex over the files, and see if one matches or not. - NOTE: this does not return the matches, just if a match occurred.""" - - if nocase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) + cregex = re.compile (regex) - for f in files: - if cregex.match (f): - return True - - return False + for f in files: + if cregex.match (f): + return True -def is_oldrar (files): - return generic_matcher (files, '^.*\.r00$') + return False -def is_newrar (files): - return generic_matcher (files, '^.*\.part0*1\.rar$') + def is_oldrar (self, files): + return self.__generic_matcher (files, '^.*\.r00$') -def is_zip (files): - return generic_matcher (files, '^.*\.zip$') + def is_newrar (self, files): + return self.__generic_matcher (files, '^.*\.part0*1\.rar$') -def is_noextract (files): - # Type that needs no extraction. - # TODO: Add others ??? - return generic_matcher (files, '^.*\.001$') + def is_zip (self, files): + return self.__generic_matcher (files, '^.*\.zip$') -def printlist (li): - for f in li: - print f + def is_noextract (self, files): + # Type that needs no extraction. + # TODO: Add others ??? + return self.__generic_matcher (files, '^.*\.001$') class PAR2Set (object): @@ -285,7 +236,7 @@ class PAR2Set (object): self.dir = dir self.p2file = p2file - self.basename = get_basename (p2file) + self.basename = self.__get_basename (p2file) # Find files that match by name only self.name_matched_files = self.__find_name_matches (self.dir, self.basename) @@ -312,6 +263,23 @@ class PAR2Set (object): self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \ self.__list_eq (self.prot_matched_files, rhs.prot_matched_files) + def __get_basename (self, name): + """Strips most kinds of endings from a filename""" + + regex = config.get_value ('regular expressions', 'basename_regex') + r = re.compile (regex, re.IGNORECASE) + done = False + + while not done: + done = True + + if r.match (name): + g = r.match (name).groups() + name = g[0] + done = False + + return name + def __parse_all_par2 (self): """Searches though self.all_p2files and tries to parse at least one of them""" done = False @@ -353,12 +321,18 @@ class PAR2Set (object): self.name_matched_files = self.__find_name_matches (self.dir, self.basename) + def __should_be_joined (self, files): + regex = re.compile ('^.*\.001$', re.IGNORECASE) + for f in files: + if regex.match (f): + return True + def runCheckAndRepair (self): PAR2_CMD = config.get_value ('commands', 'par2repair') # Get set up - all_files = self.name_matched_files + self.prot_matched_files - join = is_noextract (all_files) + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) + join = self.__should_be_joined (all_files) # assemble the command # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] @@ -384,13 +358,11 @@ class PAR2Set (object): return SUCCESS def __find_deleteable_files (self): - all_files = self.name_matched_files + self.prot_matched_files + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') dregex = re.compile (DELETE_REGEX, re.IGNORECASE) - dfiles = [f for f in all_files if dregex.match (f)] - dset = set(dfiles) # to eliminate dupes - return list(dset) + return [f for f in all_files if dregex.match (f)] def __delete_list_of_files (self, dir, files, interactive=False): # Delete a list of files @@ -404,7 +376,8 @@ class PAR2Set (object): if interactive: while not done: print 'Do you want to delete the following?:' - printlist (files) + for f in files: + print f s = raw_input ('Delete [y/N]: ').upper() if s in valid_y + valid_n: @@ -431,7 +404,7 @@ class PAR2Set (object): return ret def run_all (self): - all_files = self.name_matched_files + self.prot_matched_files + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) # Repair Stage ret = self.runCheckAndRepair () @@ -441,19 +414,19 @@ class PAR2Set (object): return -ECHECK self.__update_name_matches () - all_files = self.name_matched_files + self.prot_matched_files + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) # Extraction Stage - EXTRACT_DIR = options.extract_dir - extractor = find_extraction_heads (self.dir, all_files) - ret = extractor.extract (EXTRACT_DIR) + extractor = RarslaveExtractor (self.dir, self.all_p2files, \ + self.name_matched_files, self.prot_matched_files) + ret = extractor.runExtract (options.extract_dir) if ret != SUCCESS: logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) return -EEXTRACT self.__update_name_matches () - all_files = self.name_matched_files + self.prot_matched_files + all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) # Deletion Stage ret = self.runDelete () @@ -465,7 +438,46 @@ class PAR2Set (object): logger.addMessage ('Successfully completed: %s' % self.p2file) return SUCCESS +def run_command (cmd, indir=None): + # Runs the specified command-line in the directory given (or, in the current directory + # if none is given). It returns the status code given by the application. + + pwd = os.getcwd () + + if indir != None: + assert os.path.isdir (indir) # MUST be a directory! + os.chdir (indir) + + print 'RUNNING (%s): %s' % (indir, cmd) + ret = os.system (cmd) + os.chdir (pwd) + return ret + +def full_abspath (p): + return os.path.abspath (os.path.expanduser (p)) + +def find_par2_files (files): + """Find all par2 files in the list $files""" + + PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') + regex = re.compile (PAR2_REGEX, re.IGNORECASE) + return [f for f in files if regex.match (f)] + +def find_all_par2_files (dir): + """Finds all par2 files in a directory""" + # NOTE: does NOT return absolute paths + + if not os.path.isdir (os.path.abspath (dir)): + raise ValueError # bad directory given + + dir = os.path.abspath (dir) + files = os.listdir (dir) + + return find_par2_files (files) +def no_duplicates (li): + """Removes all duplicates from a list""" + return list(set(li)) def generate_all_parsets (dir): # Generate all parsets in the given directory. -- 2.25.1