X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=7c1b4c8ea12fda90eb7c9e3a346734a9d9ca3a85;hp=bfed51f6bf7e26efa640289ada532cdd1f0a5f90;hb=f66165cb39c650eb1cec0b24e07940760d602ffa;hpb=2a46cbb79142c6a829bbbc6b97cdb319c88607ec diff --git a/rarslave.py b/rarslave.py old mode 100644 new mode 100755 index bfed51f..7c1b4c8 --- a/rarslave.py +++ b/rarslave.py @@ -1,231 +1,98 @@ #!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=112 : +# vim: set ts=4 sts=4 sw=4 textwidth=92: + +""" +The main program of the rarslave project. + +This handles all of the commandline, configuration file, and option +work. It gets the environment set up for a run using the RarslaveDetector +class. +""" + +__author__ = "Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" +__license__ = "GNU GPL v2 (or, at your option, any later version)" + +# rarslave.py -- a usenet autorepair and autoextract utility +# +# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA VERSION="2.0.0" -PROGRAM="rarslave2" +PROGRAM="rarslave" -import re, os, sys, optparse -import Par2Parser -import RarslaveConfig -import RarslaveLogger +import os, sys, optparse, logging +import rsutil +import RarslaveDetector -# Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) -(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) -config = RarslaveConfig.RarslaveConfig() -logger = RarslaveLogger.RarslaveLogger () +# Global options from the rsutil.globals class +options = rsutil.globals.options +config = rsutil.globals.config -# Global options to be set / used later. -options = None +# A tiny class to hold logging output until we're finished +class DelayedLogger (object): -class RarslaveExtractor (object): + """A small class to hold logging output until the program is finished running. + It emulates sys.stdout in the needed ways for the logging module.""" - def __init__ (self, type): - self.type = type - self.heads = [] + def __init__ (self, output=sys.stdout.write): + self.__messages = [] + self.__output = output - def addHead (self, dir, head): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, head)) + def write (self, msg): + self.__messages.append (msg) - full_head = os.path.join (dir, head) - logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug) - self.heads.append (full_head) + def flush (self): + pass - def extract (self, todir=None): - # Extract all heads of this set + def size (self): + """Returns the number of messages queued for printing""" + return len (self.__messages) - # Create the directory $todir if it doesn't exist - if todir != None and not os.path.isdir (todir): - logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose) - try: - os.makedirs (todir) - except OSError: - logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT + def close (self): + """Print all messages, clear the queue""" + for m in self.__messages: + self.__output (m) - # Extract all heads - extraction_func = \ - { TYPE_OLDRAR : self.__extract_rar, - TYPE_NEWRAR : self.__extract_rar, - TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + self.__messages = [] - # Call the extraction function on each head - for h in self.heads: - if todir == None: - # Run in the head's directory - ret = extraction_func (h, os.path.dirname (h)) - else: - ret = extraction_func (h, todir) +# A tiny class used to find unique PAR2 sets +class CompareSet (object): - logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) + """A small class used to find unique PAR2 sets""" - # Check error code - if ret != SUCCESS: - logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - return SUCCESS - - def __extract_rar (self, file, todir): - assert os.path.isfile (file) - assert os.path.isdir (todir) - - RAR_CMD = config.get_value ('commands', 'unrar') - - cmd = '%s \"%s\"' % (RAR_CMD, file) - ret = run_command (cmd, todir) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_zip (self, file, todir): - ZIP_CMD = config.get_value ('commands', 'unzip') - - cmd = ZIP_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - def __extract_noextract (self, file, todir): - # Just move this file to the $todir, since no extraction is needed - # FIXME: NOTE: mv will fail by itself if you're moving to the same dir! - NOEXTRACT_CMD = config.get_value ('commands', 'noextract') - - # Make sure that both files are not the same file. If they are, don't run at all. - if os.path.samefile (file, os.path.join (todir, file)): - return SUCCESS - - cmd = NOEXTRACT_CMD % (file, todir) - ret = run_command (cmd) - - # Check error code - if ret != 0: - return -EEXTRACT - - return SUCCESS - - - -class RarslaveRepairer (object): - # Verify (and repair) the set - # Make sure it worked, otherwise clean up and return failure - - def __init__ (self, dir, file, join=False): - self.dir = dir # the directory containing the par2 file - self.file = file # the par2 file - self.join = join # True if the par2 set is 001 002 ... - - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) - - def checkAndRepair (self): - # Form the command: - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - PAR2_CMD = config.get_value ('commands', 'par2repair') - - # Get set up - basename = get_basename (self.file) - all_files = find_likely_files (self.dir, self.file) - all_files.sort () - par2_files = find_par2_files (all_files) - - # assemble the command - command = "%s \"%s\" " % (PAR2_CMD, self.file) - - for f in par2_files: - if f != self.file: - command += "\"%s\" " % os.path.split (f)[1] - - if self.join: - for f in all_files: - if f not in par2_files: - command += "\"%s\" " % os.path.split (f)[1] - - # run the command - ret = run_command (command, self.dir) - - # check the result - if ret != 0: - logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - return SUCCESS - -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. - - pwd = os.getcwd () - - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (indir) - - ret = os.system (cmd) - os.chdir (pwd) - return ret - -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) - -def get_basename (name): - """Strips most kinds of endings from a filename""" - - regex = config.get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False - - while not done: - done = True - - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False - - return name - -def find_likely_files (dir, p2file): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" - - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, p2file)) - - basename = get_basename (p2file) + def __init__ (self, dir, p2file): + self.dir = dir + self.p2file = p2file - dir = os.path.abspath (dir) - ename = re.escape (basename) - regex = re.compile ('^%s.*$' % (ename, )) + self.basename = rsutil.common.get_basename (self.p2file) + self.name_matches = rsutil.common.find_name_matches (self.dir, self.basename) - name_matches = [f for f in os.listdir (dir) if regex.match (f)] - try: - parsed_matches = par2parser.get_protected_files (dir, p2file) - except EnvironmentError: - parsed_matches = [] - logger.addMessage ('Bad par2 file: %s' % p2file, RarslaveLogger.MessageType.Fatal) + def __eq__ (self, rhs): + return (self.dir == rhs.dir) \ + and (self.basename == rhs.basename) \ + and rsutil.common.list_eq (self.name_matches, rhs.name_matches) - return name_matches + parsed_matches -def find_par2_files (files): - """Find all par2 files in the list $files""" +def find_all_par2_files (dir): + """Finds all par2 files in the given directory. - PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] + dir -- the directory in which to search for PAR2 files -def find_all_par2_files (dir): - """Finds all par2 files in a directory""" - # NOTE: does NOT return absolute paths + NOTE: does not return absolute paths""" if not os.path.isdir (os.path.abspath (dir)): raise ValueError # bad directory given @@ -233,214 +100,12 @@ def find_all_par2_files (dir): dir = os.path.abspath (dir) files = os.listdir (dir) - return find_par2_files (files) - -def find_extraction_heads (dir, files): - """Takes a list of possible files and finds likely heads of - extraction.""" - - # NOTE: perhaps this should happen AFTER repair is - # NOTE: successful. That way all files would already exist - - # According to various sources online: - # 1) pre rar-3.0: .rar .r00 .r01 ... - # 2) post rar-3.0: .part01.rar .part02.rar - # 3) zip all ver: .zip - - extractor = None - p2files = find_par2_files (files) - - # Old RAR type, find all files ending in .rar - if is_oldrar (files): - extractor = RarslaveExtractor (TYPE_OLDRAR) - regex = re.compile ('^.*\.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_newrar (files): - extractor = RarslaveExtractor (TYPE_NEWRAR) - regex = re.compile ('^.*\.part0*1.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_zip (files): - extractor = RarslaveExtractor (TYPE_ZIP) - regex = re.compile ('^.*\.zip$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_noextract (files): - # Use the Par2 Parser (from cfv) here to find out what files are protected. - # Since these are not being extracted, they will be mv'd to another directory - # later. - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - for f in p2files: - done = False - try: - prot_files = par2parser.get_protected_files (dir, f) - done = True - except EnvironmentError: - logger.addMessage ('Error parsing PAR2 file: %s', f) - continue - - if done: - break - - if done: - for f in prot_files: - extractor.addHead (dir, f) - else: - logger.addMessage ('Error parsing all PAR2 files in this set ...') - - # Make sure we found the type - if extractor == None: - logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0], - RarslaveLogger.MessageType.Verbose) - - # No-heads here, but it's better than failing completely - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - return extractor - -def generic_matcher (files, regex, nocase=False): - """Run the regex over the files, and see if one matches or not. - NOTE: this does not return the matches, just if a match occurred.""" - - if nocase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - for f in files: - if cregex.match (f): - return True - - return False - -def is_oldrar (files): - return generic_matcher (files, '^.*\.r00$') - -def is_newrar (files): - return generic_matcher (files, '^.*\.part0*1\.rar$') - -def is_zip (files): - return generic_matcher (files, '^.*\.zip$') - -def is_noextract (files): - # Type that needs no extraction. - # TODO: Add others ??? - return generic_matcher (files, '^.*\.001$') - -def find_deleteable_files (files): - # Deleteable types regex should come from the config - dfiles = [] - DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) - - return [f for f in files if dregex.match (f)] - -def printlist (li): - for f in li: - print f - -class PAR2Set (object): - - dir = None - file = None - likely_files = [] - - def __init__ (self, dir, file): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) - - self.dir = dir - self.file = file - - basename = get_basename (file) - self.likely_files = find_likely_files (dir, file) - - def __list_eq (self, l1, l2): - - if len(l1) != len(l2): - return False - - for e in l1: - if e not in l2: - return False - - return True - - def __eq__ (self, rhs): - return self.__list_eq (self.likely_files, rhs.likely_files) - - def run_all (self): - par2files = find_par2_files (self.likely_files) - par2head = par2files[0] - - join = is_noextract (self.likely_files) - - # Repair Stage - repairer = RarslaveRepairer (self.dir, par2head, join) - ret = repairer.checkAndRepair () - - if ret != SUCCESS: - logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) - return -ECHECK - - # Extraction Stage - EXTRACT_DIR = options.extract_dir - extractor = find_extraction_heads (self.dir, self.likely_files) - ret = extractor.extract (EXTRACT_DIR) - - if ret != SUCCESS: - logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) - return -EEXTRACT - - # Deletion Stage - DELETE_INTERACTIVE = options.interactive - deleteable_files = find_deleteable_files (self.likely_files) - ret = delete_list (self.dir, deleteable_files, DELETE_INTERACTIVE) - - if ret != SUCCESS: - logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) - return -EDELETE - - logger.addMessage ('Successfully completed: %s' % par2head) - return SUCCESS - -def delete_list (dir, files, interactive=False): - # Delete a list of files - - assert os.path.isdir (dir) - - done = False - valid_y = ['Y', 'YES'] - valid_n = ['N', 'NO'] - - if interactive: - while not done: - print 'Do you want to delete the following?:' - printlist (files) - s = raw_input ('Delete [y/N]: ').upper() - - if s in valid_y + valid_n: - done = True - - if s in valid_n: - return SUCCESS - - for f in files: - os.remove (os.path.join (dir, f)) - - return SUCCESS - + return rsutil.common.find_par2_files (files) def generate_all_parsets (dir): - # Generate all parsets in the given directory. + """Generate all parsets in the given directory + + dir -- the directory in which to search""" assert os.path.isdir (dir) # Directory MUST be valid @@ -448,11 +113,11 @@ def generate_all_parsets (dir): p2files = find_all_par2_files (dir) for f in p2files: - p = PAR2Set (dir, f) + p = CompareSet (dir, f) if p not in parsets: parsets.append (p) - return parsets + return [(p.dir, p.p2file) for p in parsets] def check_required_progs(): """Check if the required programs are installed""" @@ -460,13 +125,13 @@ def check_required_progs(): shell_not_found = 32512 needed = [] - if run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found: + if rsutil.common.run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found: needed.append ('par2repair') - if run_command ('unrar --help > /dev/null 2>&1') == shell_not_found: + if rsutil.common.run_command ('unrar --help > /dev/null 2>&1') == shell_not_found: needed.append ('unrar') - if run_command ('unzip --help > /dev/null 2>&1') == shell_not_found: + if rsutil.common.run_command ('unzip --help > /dev/null 2>&1') == shell_not_found: needed.append ('unzip') if needed: @@ -476,9 +141,11 @@ def check_required_progs(): sys.exit(1) def run_options (options): + """Process all of the commandline options, doing thing such as printing the + version number, etc.""" # Fix directories - options.work_dir = full_abspath (options.work_dir) + options.work_dir = rsutil.common.full_abspath (options.work_dir) # Make sure that the directory is valid if not os.path.isdir (options.work_dir): @@ -488,7 +155,7 @@ def run_options (options): sys.exit (1) if options.extract_dir != None: - options.extract_dir = full_abspath (options.extract_dir) + options.extract_dir = rsutil.common.full_abspath (options.extract_dir) if options.version: print PROGRAM + ' - ' + VERSION @@ -505,68 +172,51 @@ def run_options (options): if options.write_def_config: config.write_config (default=True) + sys.exit (0) if options.write_config: config.write_config () + sys.exit (0) def find_loglevel (options): + """Find the log level that should be printed by the logging class""" loglevel = options.verbose - options.quiet - if loglevel < RarslaveLogger.MessageType.Fatal: - loglevel = RarslaveLogger.MessageType.Fatal - - if loglevel > RarslaveLogger.MessageType.Debug: - loglevel = RarslaveLogger.MessageType.Debug - - return loglevel - -def printMessageTable (loglevel): + if loglevel > 1: + loglevel = 1 - if logger.hasFatalMessages (): - print '\nFatal Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Fatal) + if loglevel < -3: + loglevel = -3 - if loglevel == RarslaveLogger.MessageType.Fatal: - return + LEVELS = { 1 : logging.DEBUG, + 0 : logging.INFO, + -1: logging.WARNING, + -2: logging.ERROR, + -3: logging.CRITICAL + } - if logger.hasNormalMessages (): - print '\nNormal Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Normal) - - if loglevel == RarslaveLogger.MessageType.Normal: - return - - if logger.hasVerboseMessages (): - print '\nVerbose Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Verbose) - - if loglevel == RarslaveLogger.MessageType.Verbose: - return - - if logger.hasDebugMessages (): - print '\nDebug Messages\n' + '=' * 80 - logger.printLoglevel (RarslaveLogger.MessageType.Debug) - - return + return LEVELS [loglevel] def main (): + # Setup the logger + logger = DelayedLogger () + logging.basicConfig (stream=logger, level=logging.WARNING, \ + format='%(levelname)-8s %(message)s') + # Build the OptionParser parser = optparse.OptionParser() - parser.add_option('-n', '--not-recursive', - action='store_false', dest='recursive', - default=config.get_value('options', 'recursive'), + parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive', + default=rsutil.common.config_get_value('options', 'recursive'), help="Don't run recursively") - parser.add_option('-d', '--work-dir', - dest='work_dir', type='string', - default=config.get_value('directories', 'working_directory'), + parser.add_option('-d', '--work-dir', dest='work_dir', type='string', + default=rsutil.common.config_get_value('directories', 'working_directory'), help="Start running at DIR", metavar='DIR') - parser.add_option('-e', '--extract-dir', - dest='extract_dir', type='string', - default=config.get_value('directories', 'extract_directory'), + parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string', + default=rsutil.common.config_get_value('directories', 'extract_directory'), help="Extract to DIR", metavar='DIR') parser.add_option('-p', '--check-required-programs', @@ -583,7 +233,7 @@ def main (): default=False, help="Write out the current config") parser.add_option('-i', '--interactive', dest='interactive', action='store_true', - default=config.get_value('options', 'interactive'), + default=rsutil.common.config_get_value('options', 'interactive'), help="Confirm before removing files") parser.add_option('-q', '--quiet', dest='quiet', action='count', @@ -599,29 +249,34 @@ def main (): # Parse the given options global options - (options, args) = parser.parse_args() + (rsutil.globals.options, args) = parser.parse_args() + options = rsutil.globals.options # Run any special actions that are needed on these options run_options (options) - # Find the loglevel using the options given - loglevel = find_loglevel (options) + # Find the loglevel using the options given + logging.getLogger().setLevel (find_loglevel (options)) # Run recursively if options.recursive: for (dir, subdirs, files) in os.walk (options.work_dir): parsets = generate_all_parsets (dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Non-recursive else: parsets = generate_all_parsets (options.work_dir) - for p in parsets: - p.run_all () + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () # Print the results - printMessageTable (loglevel) + if logger.size () > 0: + print '\nLog\n' + '=' * 80 + logger.close () # Done! return 0