X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=657fa7247e262701863720e8a762f1916461893b;hp=7e3881628a44c7ee339f2cb8b488a0622d49f3a1;hb=f1d2c2eb0d60addd6e99e7877a9e77ee1223f674;hpb=aebb7adb2c800b1efc7c7d1877a16609cfe76b65 diff --git a/rarslave.py b/rarslave.py index 7e38816..657fa72 100644 --- a/rarslave.py +++ b/rarslave.py @@ -1,12 +1,184 @@ #!/usr/bin/env python # vim: set ts=4 sts=4 sw=4 textwidth=112 : -import re, os, sys +VERSION="2.0.0" +PROGRAM="rarslave2" + +import re, os, sys, optparse +import par2parser +import RarslaveConfig +import RarslaveLogger + +# Global Variables +(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) +(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4) +config = RarslaveConfig.RarslaveConfig() +logger = RarslaveLogger.RarslaveLogger () + +# Global options to be set / used later. +options = None + +class RarslaveExtractor (object): + + def __init__ (self, type): + self.type = type + self.heads = [] + + def addHead (self, dir, head): + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, head)) + + full_head = os.path.join (dir, head) + logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug) + self.heads.append (full_head) + + def extract (self, todir=None): + # Extract all heads of this set + + # Create the directory $todir if it doesn't exist + if todir != None and not os.path.isdir (todir): + logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose) + try: + os.makedirs (todir) + except OSError: + logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + # Extract all heads + extraction_func = \ + { TYPE_OLDRAR : self.__extract_rar, + TYPE_NEWRAR : self.__extract_rar, + TYPE_ZIP : self.__extract_zip, + TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + + # Call the extraction function on each head + for h in self.heads: + if todir == None: + # Run in the head's directory + ret = extraction_func (h, os.path.dirname (h)) + else: + ret = extraction_func (h, todir) + + logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug) + + # Check error code + if ret != SUCCESS: + logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + return SUCCESS + + def __extract_rar (self, file, todir): + assert os.path.isfile (file) + assert os.path.isdir (todir) + + RAR_CMD = config.get_value ('commands', 'unrar') + + cmd = '%s \"%s\"' % (RAR_CMD, file) + ret = run_command (cmd, todir) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + + def __extract_zip (self, file, todir): + ZIP_CMD = config.get_value ('commands', 'unzip') + + cmd = ZIP_CMD % (file, todir) + ret = run_command (cmd) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + + def __extract_noextract (self, file, todir): + # Just move this file to the $todir, since no extraction is needed + # FIXME: NOTE: mv will fail by itself if you're moving to the same dir! + NOEXTRACT_CMD = config.get_value ('commands', 'noextract') + + # Make sure that both files are not the same file. If they are, don't run at all. + if os.path.samefile (file, os.path.join (todir, file)): + return SUCCESS + + cmd = NOEXTRACT_CMD % (file, todir) + ret = run_command (cmd) + + # Check error code + if ret != 0: + return -EEXTRACT + + return SUCCESS + + + +class RarslaveRepairer (object): + # Verify (and repair) the set + # Make sure it worked, otherwise clean up and return failure + + def __init__ (self, dir, file, join=False): + self.dir = dir # the directory containing the par2 file + self.file = file # the par2 file + self.join = join # True if the par2 set is 001 002 ... + + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, file)) + + def checkAndRepair (self): + # Form the command: + # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] + PAR2_CMD = config.get_value ('commands', 'par2repair') + + # Get set up + basename = get_basename (self.file) + all_files = find_likely_files (basename, self.dir) + all_files.sort () + par2_files = find_par2_files (all_files) + + # assemble the command + command = "%s \"%s\" " % (PAR2_CMD, self.file) + + for f in par2_files: + if f != self.file: + command += "\"%s\" " % os.path.split (f)[1] + + if self.join: + for f in all_files: + if f not in par2_files: + command += "\"%s\" " % os.path.split (f)[1] + + # run the command + ret = run_command (command, self.dir) + + # check the result + if ret != 0: + logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal) + return -ECHECK + + return SUCCESS + +def run_command (cmd, indir=None): + # Runs the specified command-line in the directory given (or, in the current directory + # if none is given). It returns the status code given by the application. + + pwd = os.getcwd () + + if indir != None: + assert os.path.isdir (indir) # MUST be a directory! + os.chdir (indir) + + return os.system (cmd) + +def full_abspath (p): + return os.path.abspath (os.path.expanduser (p)) def get_basename (name): """Strips most kinds of endings from a filename""" - regex = '^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$' + regex = config.get_value ('regular expressions', 'basename_regex') r = re.compile (regex, re.IGNORECASE) done = False @@ -33,22 +205,424 @@ def find_likely_files (name, dir): return [f for f in os.listdir (dir) if regex.match (f)] +def find_par2_files (files): + """Find all par2 files in the list $files""" + + PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex') + regex = re.compile (PAR2_REGEX, re.IGNORECASE) + return [f for f in files if regex.match (f)] + def find_all_par2_files (dir): """Finds all par2 files in a directory""" + # NOTE: does NOT return absolute paths if not os.path.isdir (os.path.abspath (dir)): raise ValueError # bad directory given dir = os.path.abspath (dir) - regex = re.compile ('^.*\.par2$', re.IGNORECASE) + files = os.listdir (dir) - # Find all files - return [f for f in os.listdir (dir) if regex.match (f)] + return find_par2_files (files) + +def has_extension (f, ext): + """Checks if f has the extension ext""" + + if ext[0] != '.': + ext = '.' + ext + + ext = re.escape (ext) + regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE) + return regex.match (f) + +def find_extraction_heads (dir, files): + """Takes a list of possible files and finds likely heads of + extraction.""" + + # NOTE: perhaps this should happen AFTER repair is + # NOTE: successful. That way all files would already exist + + # According to various sources online: + # 1) pre rar-3.0: .rar .r00 .r01 ... + # 2) post rar-3.0: .part01.rar .part02.rar + # 3) zip all ver: .zip + + extractor = None + p2files = find_par2_files (files) + + # Old RAR type, find all files ending in .rar + if is_oldrar (files): + extractor = RarslaveExtractor (TYPE_OLDRAR) + regex = re.compile ('^.*\.rar$', re.IGNORECASE) + for f in files: + if regex.match (f): + extractor.addHead (dir, f) + + if is_newrar (files): + extractor = RarslaveExtractor (TYPE_NEWRAR) + regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE) + for f in files: + if regex.match (f): + extractor.addHead (dir, f) + + if is_zip (files): + extractor = RarslaveExtractor (TYPE_ZIP) + regex = re.compile ('^.*\.zip$', re.IGNORECASE) + for f in files: + if regex.match (f): + extractor.addHead (dir, f) + + if is_noextract (files): + # Use the Par2 Parser (from cfv) here to find out what files are protected. + # Since these are not being extracted, they will be mv'd to another directory + # later. + extractor = RarslaveExtractor (TYPE_NOEXTRACT) + + for f in p2files: + done = False + try: + prot_files = par2parser.get_protected_files (dir, f) + done = True + except: #FIXME: add the actual exceptions + logger.addMessage ('Error parsing PAR2 file: %s', f) + continue + + if done: + break + + if done: + for f in prot_files: + extractor.addHead (dir, f) + else: + logger.addMessage ('Error parsing all PAR2 files in this set ...') + + # Make sure we found the type + if extractor == None: + logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0], + RarslaveLogger.MessageType.Fatal) + + # No-heads here, but it's better than failing completely + extractor = RarslaveExtractor (TYPE_NOEXTRACT) + + return extractor + +def is_oldrar (files): + for f in files: + if has_extension (f, '.r00'): + return True + + return False + +def is_newrar (files): + for f in files: + if has_extension (f, '.part01.rar'): + return True + + return False + +def is_zip (files): + for f in files: + if has_extension (f, '.zip'): + return True + + return False + +def is_noextract (files): + # Type that needs no extraction. + # TODO: Add others ??? + for f in files: + if has_extension (f, '.001'): + return True + + return False + +def find_deleteable_files (files): + # Deleteable types regex should come from the config + dfiles = [] + DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex') + dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + + return [f for f in files if dregex.match (f)] + +def printlist (li): + for f in li: + print f + +class PAR2Set (object): + dir = None + file = None + likely_files = [] + def __init__ (self, dir, file): + assert os.path.isdir (dir) + assert os.path.isfile (os.path.join (dir, file)) + + self.dir = dir + self.file = file + + basename = get_basename (file) + self.likely_files = find_likely_files (basename, dir) + + def __list_eq (self, l1, l2): + + if len(l1) != len(l2): + return False + + for e in l1: + if e not in l2: + return False + + return True + + def __eq__ (self, rhs): + return self.__list_eq (self.likely_files, rhs.likely_files) + + def run_all (self): + par2files = find_par2_files (self.likely_files) + par2head = par2files[0] + + join = is_noextract (self.likely_files) + + # Repair Stage + repairer = RarslaveRepairer (self.dir, par2head, join) + ret = repairer.checkAndRepair () + + if ret != SUCCESS: + logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + return -ECHECK + + # Extraction Stage + EXTRACT_DIR = options.extract_dir + extractor = find_extraction_heads (self.dir, self.likely_files) + ret = extractor.extract (EXTRACT_DIR) + + if ret != SUCCESS: + logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + return -EEXTRACT + + # Deletion Stage + DELETE_INTERACTIVE = options.interactive + deleteable_files = find_deleteable_files (self.likely_files) + ret = delete_list (deleteable_files, DELETE_INTERACTIVE) + + if ret != SUCCESS: + logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal) + return -EDELETE + + logger.addMessage ('Successfully completed: %s' % par2head) + return SUCCESS + +def delete_list (files, interactive=False): + # Delete a list of files + + done = False + valid_y = ['Y', 'YES'] + valid_n = ['N', 'NO'] + + if interactive: + while not done: + print 'Do you want to delete the following?:' + s = raw_input ('Delete [y/N]: ').upper() + + if s in valid_y + valid_n: + done = True + + if s in valid_n: + return SUCCESS + + for f in files: + os.remove (f) + + return SUCCESS + + +def generate_all_parsets (dir): + # Generate all parsets in the given directory. + + assert os.path.isdir (dir) # Directory MUST be valid + + parsets = [] + p2files = find_all_par2_files (dir) + + for f in p2files: + p = PAR2Set (dir, f) + if p not in parsets: + parsets.append (p) + + return parsets + +def check_required_progs(): + """Check if the required programs are installed""" + + shell_not_found = 32512 + needed = [] + + if run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found: + needed.append ('par2repair') + + if run_command ('unrar --help > /dev/null 2>&1') == shell_not_found: + needed.append ('unrar') + + if run_command ('unzip --help > /dev/null 2>&1') == shell_not_found: + needed.append ('unzip') + + if needed: + for n in needed: + print 'Needed program "%s" not found in $PATH' % (n, ) + + sys.exit(1) + +def run_options (options): + + # Fix directories + options.work_dir = full_abspath (options.work_dir) + + # Make sure that the directory is valid + if not os.path.isdir (options.work_dir): + sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir) + sys.stderr.write ('option to override the working directory temporarily, or edit the\n') + sys.stderr.write ('configuration file to override the working directory permanently.\n') + sys.exit (1) + + if options.extract_dir != None: + options.extract_dir = full_abspath (options.extract_dir) + + if options.version: + print PROGRAM + ' - ' + VERSION + print + print 'Copyright (c) 2005,2006 Ira W. Snyder (devel@irasnyder.com)' + print + print 'This program comes with ABSOLUTELY NO WARRANTY.' + print 'This is free software, and you are welcome to redistribute it' + print 'under certain conditions. See the file COPYING for details.' + sys.exit (0) + + if options.check_progs: + check_required_progs () + + if options.write_def_config: + config.write_config (default=True) + + if options.write_config: + config.write_config () + +def find_loglevel (options): + + loglevel = options.verbose - options.quiet + + if loglevel < RarslaveLogger.MessageType.Fatal: + loglevel = RarslaveLogger.MessageType.Fatal + + if loglevel > RarslaveLogger.MessageType.Debug: + loglevel = RarslaveLogger.MessageType.Debug + + return loglevel + +def printMessageTable (loglevel): + + if logger.hasFatalMessages (): + print '\nFatal Messages\n' + '=' * 80 + logger.printLoglevel (RarslaveLogger.MessageType.Fatal) + + if loglevel == RarslaveLogger.MessageType.Fatal: + return + + if logger.hasNormalMessages (): + print '\nNormal Messages\n' + '=' * 80 + logger.printLoglevel (RarslaveLogger.MessageType.Normal) + + if loglevel == RarslaveLogger.MessageType.Normal: + return + + if logger.hasVerboseMessages (): + print '\nVerbose Messages\n' + '=' * 80 + logger.printLoglevel (RarslaveLogger.MessageType.Verbose) + + if loglevel == RarslaveLogger.MessageType.Verbose: + return + + if logger.hasDebugMessages (): + print '\nDebug Messages\n' + '=' * 80 + logger.printLoglevel (RarslaveLogger.MessageType.Debug) + + return def main (): - print find_all_par2_files ('/home/irasnyd/downloads/test_material/01/') + + # Build the OptionParser + parser = optparse.OptionParser() + parser.add_option('-n', '--not-recursive', + action='store_false', dest='recursive', + default=config.get_value('options', 'recursive'), + help="Don't run recursively") + + parser.add_option('-d', '--work-dir', + dest='work_dir', type='string', + default=config.get_value('directories', 'working_directory'), + help="Start running at DIR", metavar='DIR') + + parser.add_option('-e', '--extract-dir', + dest='extract_dir', type='string', + default=config.get_value('directories', 'extract_directory'), + help="Extract to DIR", metavar='DIR') + + parser.add_option('-p', '--check-required-programs', + action='store_true', dest='check_progs', + default=False, + help="Check for required programs") + + parser.add_option('-f', '--write-default-config', + action='store_true', dest='write_def_config', + default=False, help="Write out a new default config") + + parser.add_option('-c', '--write-new-config', + action='store_true', dest='write_config', + default=False, help="Write out the current config") + + parser.add_option('-i', '--interactive', dest='interactive', action='store_true', + default=config.get_value('options', 'interactive'), + help="Confirm before removing files") + + parser.add_option('-q', '--quiet', dest='quiet', action='count', + default=0, help="Output fatal messages only") + + parser.add_option('-v', '--verbose', dest='verbose', action='count', + default=0, help="Output extra information") + + parser.add_option('-V', '--version', dest='version', action='store_true', + default=False, help="Output version information") + + parser.version = VERSION + + # Parse the given options + global options + (options, args) = parser.parse_args() + + # Run any special actions that are needed on these options + run_options (options) + + # Find the loglevel using the options given + loglevel = find_loglevel (options) + + # Run recursively + if options.recursive: + for (dir, subdirs, files) in os.walk (options.work_dir): + parsets = generate_all_parsets (dir) + for p in parsets: + p.run_all () + + # Non-recursive + else: + parsets = generate_all_parsets (options.work_dir) + for p in parsets: + p.run_all () + + # Print the results + printMessageTable (loglevel) + + # Done! + return 0 if __name__ == '__main__': main () +