X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=a18e6a3fd7f518ed30fcbc35d78b1b4f63547752;hp=6972d411358b1d6f32524fda50f93fe8bf59404f;hb=98204d147a599a273e4ab0a85caab8f8287affb3;hpb=17de3f5e52714bcdad168416ea14b2aebb5db887 diff --git a/rarslave.py b/rarslave.py index 6972d41..a18e6a3 100644 --- a/rarslave.py +++ b/rarslave.py @@ -4,464 +4,53 @@ VERSION="2.0.0" PROGRAM="rarslave2" -import re, os, sys, optparse -import Par2Parser -import RarslaveConfig -import RarslaveLogger +import os, sys, optparse, logging +import RarslaveDetector +import RarslaveGlobals +from RarslaveCommon import * -# Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5) -(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) -config = RarslaveConfig.RarslaveConfig() -logger = RarslaveLogger.RarslaveLogger () +# Global options from the RarslaveGlobals class +options = RarslaveGlobals.options +config = RarslaveGlobals.config -# Global options to be set / used later. -options = None +# A tiny class to hold logging output until we're finished +class DelayedLogger (object): + def __init__ (self, output=sys.stdout.write): + self.__messages = [] + self.__output = output -class RarslaveExtractor (object): + def write (self, msg): + self.__messages.append (msg) - type = None - heads = [] + def flush (self): + pass - def __init__ (self, dir, p2files, name_files, prot_files): + def size (self): + """Returns the number of messages queued for printing""" + return len (self.__messages) - self.dir = dir - self.p2files = p2files - self.name_matched_files = name_files - self.prot_matched_files = prot_files - - # Find the type - self.type = self.__find_type () - - logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug) - - # Find the heads - self.heads = self.__find_heads () - - for h in self.heads: - logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug) - - def __repr__ (self): - return \ - { TYPE_OLDRAR : 'Old RAR', - TYPE_NEWRAR : 'New RAR', - TYPE_ZIP : 'Zip', - TYPE_NOEXTRACT : 'No Extract', - TYPE_UNKNOWN : 'Unknown' } [self.type] - - def __find_type (self): - - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - if self.is_oldrar (all_files): - return TYPE_OLDRAR - elif self.is_newrar (all_files): - return TYPE_NEWRAR - elif self.is_zip (all_files): - return TYPE_ZIP - elif self.is_noextract (all_files): - return TYPE_NOEXTRACT - - return TYPE_UNKNOWN - - def __generic_find_heads (self, regex, ignorecase=True): - - heads = [] - - if ignorecase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - for f in all_files: - if cregex.match (f): - heads.append (f) - - return heads - - def __find_heads (self): - - if self.type == TYPE_OLDRAR: - return self.__generic_find_heads ('^.*\.rar$') - elif self.type == TYPE_NEWRAR: - return self.__generic_find_heads ('^.*\.part0*1\.rar$') - elif self.type == TYPE_ZIP: - return self.__generic_find_heads ('^.*\.zip$') - elif self.type == TYPE_NOEXTRACT: - return self.prot_matched_files - - return [] - - def __create_directory (self, dir): - if dir == None: - return SUCCESS - - if os.path.isdir (dir): - return SUCCESS - - try: - os.makedirs (dir) - logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - return SUCCESS - - def runExtract (self, todir=None): - # Extract all heads of this set - - # Extract to the head's dir if we don't care where to extract - if todir == None: - todir = self.dir - - # Create the directory $todir if it doesn't exist - ret = self.__create_directory (todir) - - if ret != SUCCESS: - return -EEXTRACT - - # Extract all heads - extraction_func = \ - { TYPE_OLDRAR : self.__extract_rar, - TYPE_NEWRAR : self.__extract_rar, - TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract, - TYPE_UNKNOWN : self.__extract_unknown }[self.type] - - # Call the extraction function on each head - for h in self.heads: - full_head = full_abspath (h) - ret = extraction_func (full_head, todir) - logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) - - # Check error code - if ret != SUCCESS: - logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - return SUCCESS - - def __extract_rar (self, file, todir): - assert os.path.isfile (file) - assert os.path.isdir (todir) + def close (self): + """Print all messages, clear the queue""" + for m in self.__messages: + self.__output (m) - RAR_CMD = config.get_value ('commands', 'unrar') + self.__messages = [] - cmd = '%s \"%s\"' % (RAR_CMD, file) - ret = run_command (cmd, todir) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_zip (self, file, todir): - ZIP_CMD = config.get_value ('commands', 'unzip') - - cmd = ZIP_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_noextract (self, file, todir): - # Just move this file to the $todir, since no extraction is needed - # FIXME: NOTE: mv will fail by itself if you're moving to the same dir! - NOEXTRACT_CMD = config.get_value ('commands', 'noextract') - - # Make sure that both files are not the same file. If they are, don't run at all. - if os.path.samefile (file, os.path.join (todir, file)): - return SUCCESS - - cmd = NOEXTRACT_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_unknown (self, file, todir): - return SUCCESS - - def __generic_matcher (self, files, regex, nocase=False): - """Run the regex over the files, and see if one matches or not. - NOTE: this does not return the matches, just if a match occurred.""" - - if nocase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - for f in files: - if cregex.match (f): - return True - - return False - - def is_oldrar (self, files): - return self.__generic_matcher (files, '^.*\.r00$') - - def is_newrar (self, files): - return self.__generic_matcher (files, '^.*\.part0*1\.rar$') - - def is_zip (self, files): - return self.__generic_matcher (files, '^.*\.zip$') - - def is_noextract (self, files): - # Type that needs no extraction. - # TODO: Add others ??? - return self.__generic_matcher (files, '^.*\.001$') - -class PAR2Set (object): - - dir = None - p2file = None # The starting par2 - basename = None # The p2file's basename - all_p2files = [] - name_matched_files = [] # Files that match by basename of the p2file - prot_matched_files = [] # Files that match by being protected members +# A tiny class used to find unique PAR2 sets +class CompareSet (object): def __init__ (self, dir, p2file): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, p2file)) - self.dir = dir self.p2file = p2file - self.basename = self.__get_basename (p2file) - - # Find files that match by name only - self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - # Find all par2 files for this set using name matches - self.all_p2files = find_par2_files (self.name_matched_files) - - # Try to get the protected files for this set - self.prot_matched_files = self.__parse_all_par2 () - - def __list_eq (self, l1, l2): - - if len(l1) != len(l2): - return False - - for e in l1: - if e not in l2: - return False - - return True + self.basename = get_basename (self.p2file) + self.name_matches = find_name_matches (self.dir, self.basename) def __eq__ (self, rhs): - return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ - self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \ - self.__list_eq (self.prot_matched_files, rhs.prot_matched_files) - - def __get_basename (self, name): - """Strips most kinds of endings from a filename""" - - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False + return (self.dir == rhs.dir) \ + and (self.basename == rhs.basename) \ + and list_eq (self.name_matches, rhs.name_matches) - while not done: - done = True - - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False - - return name - - def __parse_all_par2 (self): - """Searches though self.all_p2files and tries to parse at least one of them""" - done = False - files = [] - - for f in self.all_p2files: - - # Exit early if we've found a good file - if done: - break - - try: - files = Par2Parser.get_protected_files (self.dir, f) - done = True - except (EnvironmentError, OSError, OverflowError): - logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal) - - # Now that we're out of the loop, check if we really finished - if not done: - logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - - # Return whatever we've got, empty or not - return files - - def __find_name_matches (self, dir, basename): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" - - assert os.path.isdir (dir) - - ename = re.escape (basename) - regex = re.compile ('^%s.*$' % (ename, )) - - return [f for f in os.listdir (dir) if regex.match (f)] - - def __update_name_matches (self): - """Updates the self.name_matched_files variable with the most current information. - This should be called after the directory contents are likely to change.""" - - self.name_matched_files = self.__find_name_matches (self.dir, self.basename) - - def __should_be_joined (self, files): - regex = re.compile ('^.*\.001$', re.IGNORECASE) - for f in files: - if regex.match (f): - return True - - def runCheckAndRepair (self): - PAR2_CMD = config.get_value ('commands', 'par2repair') - - # Get set up - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - join = self.__should_be_joined (all_files) - - # assemble the command - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - command = "%s \"%s\" " % (PAR2_CMD, self.p2file) - - for f in self.all_p2files: - if f != self.p2file: - command += "\"%s\" " % os.path.split (f)[1] - - if join: - for f in all_files: - if f not in self.p2files: - command += "\"%s\" " % os.path.split (f)[1] - - # run the command - ret = run_command (command, self.dir) - - # check the result - if ret != 0: - logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - return SUCCESS - - def __find_deleteable_files (self): - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) - - return [f for f in all_files if dregex.match (f)] - - def __delete_list_of_files (self, dir, files, interactive=False): - # Delete a list of files - - assert os.path.isdir (dir) - - done = False - valid_y = ['Y', 'YES'] - valid_n = ['N', 'NO', ''] - - if interactive: - while not done: - print 'Do you want to delete the following?:' - for f in files: - print f - s = raw_input ('Delete [y/N]: ').upper() - - if s in valid_y + valid_n: - done = True - - if s in valid_n: - return SUCCESS - - for f in files: - try: - os.remove (os.path.join (dir, f)) - logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug) - except: - logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f), - RarslaveLogger.MessageType.Fatal) - return -EDELETE - - return SUCCESS - - def runDelete (self): - deleteable_files = self.__find_deleteable_files () - ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive) - - return ret - - def run_all (self): - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Repair Stage - ret = self.runCheckAndRepair () - - if ret != SUCCESS: - logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - self.__update_name_matches () - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Extraction Stage - extractor = RarslaveExtractor (self.dir, self.all_p2files, \ - self.name_matched_files, self.prot_matched_files) - ret = extractor.runExtract (options.extract_dir) - - if ret != SUCCESS: - logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - self.__update_name_matches () - all_files = no_duplicates (self.name_matched_files + self.prot_matched_files) - - # Deletion Stage - ret = self.runDelete () - - if ret != SUCCESS: - logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal) - return -EDELETE - - logger.addMessage ('Successfully completed: %s' % self.p2file) - return SUCCESS - -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. - - pwd = os.getcwd () - - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) - - print 'RUNNING (%s): %s' % (indir, cmd) - ret = os.system (cmd) - os.chdir (pwd) - return ret - -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) - -def find_par2_files (files): - """Find all par2 files in the list $files""" - - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] def find_all_par2_files (dir): """Finds all par2 files in a directory""" @@ -475,10 +64,6 @@ def find_all_par2_files (dir): return find_par2_files (files) -def no_duplicates (li): - """Removes all duplicates from a list""" - return list(set(li)) - def generate_all_parsets (dir): # Generate all parsets in the given directory. @@ -488,11 +73,11 @@ def generate_all_parsets (dir): p2files = find_all_par2_files (dir) for f in p2files: - p = PAR2Set (dir, f) + p = CompareSet (dir, f) if p not in parsets: parsets.append (p) - return parsets + return [(p.dir, p.p2file) for p in parsets] def check_required_progs(): """Check if the required programs are installed""" @@ -553,60 +138,43 @@ def find_loglevel (options): loglevel = options.verbose - options.quiet - if loglevel < RarslaveLogger.MessageType.Fatal: - loglevel = RarslaveLogger.MessageType.Fatal - - if loglevel > RarslaveLogger.MessageType.Debug: - loglevel = RarslaveLogger.MessageType.Debug - - return loglevel + if loglevel > 1: + loglevel = 1 -def printMessageTable (loglevel): + if loglevel < -3: + loglevel = -3 - if logger.hasFatalMessages (): - print '\nFatal Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Fatal) + LEVELS = { 1 : logging.DEBUG, + 0 : logging.INFO, + -1: logging.WARNING, + -2: logging.ERROR, + -3: logging.CRITICAL + } - if loglevel == RarslaveLogger.MessageType.Fatal: - return - - if logger.hasNormalMessages (): - print '\nNormal Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Normal) - - if loglevel == RarslaveLogger.MessageType.Normal: - return - - if logger.hasVerboseMessages (): - print '\nVerbose Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Verbose) - - if loglevel == RarslaveLogger.MessageType.Verbose: - return - - if logger.hasDebugMessages (): - print '\nDebug Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Debug) - - return + return LEVELS [loglevel] def main (): + # Setup the logger + logger = DelayedLogger () + logging.basicConfig (stream=logger, level=logging.WARNING, \ + format='%(levelname)-8s %(message)s') + # Build the OptionParser parser = optparse.OptionParser() parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive', - default=config.get_value('options', 'recursive'), + default=config_get_value('options', 'recursive'), help="Don't run recursively") parser.add_option('-d', '--work-dir', dest='work_dir', type='string', - default=config.get_value('directories', 'working_directory'), + default=config_get_value('directories', 'working_directory'), help="Start running at DIR", metavar='DIR') parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string', - default=config.get_value('directories', 'extract_directory'), + default=config_get_value('directories', 'extract_directory'), help="Extract to DIR", metavar='DIR') parser.add_option('-p', '--check-required-programs', @@ -623,7 +191,7 @@ def main (): default=False, help="Write out the current config") parser.add_option('-i', '--interactive', dest='interactive', action='store_true', - default=config.get_value('options', 'interactive'), + default=config_get_value('options', 'interactive'), help="Confirm before removing files") parser.add_option('-q', '--quiet', dest='quiet', action='count', @@ -639,29 +207,34 @@ def main (): # Parse the given options global options - (options, args) = parser.parse_args() + (RarslaveGlobals.options, args) = parser.parse_args() + options = RarslaveGlobals.options # Run any special actions that are needed on these options run_options (options) # Find the loglevel using the options given - loglevel = find_loglevel (options) + logging.getLogger().setLevel (find_loglevel (options)) # Run recursively if options.recursive: for (dir, subdirs, files) in os.walk (options.work_dir): parsets = generate_all_parsets (dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Non-recursive else: parsets = generate_all_parsets (options.work_dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Print the results - printMessageTable (loglevel) + if logger.size () > 0: + print 'Log\n' + '=' * 80 + logger.close () # Done! return 0