X-Git-Url: https://www.irasnyder.com/gitweb/?p=rarslave2.git;a=blobdiff_plain;f=rarslave.py;h=a27c334cfe2e14d2b159c2defbaf5657c81e1de4;hp=89d526dcdd12d7c0fec30eba37a8b8db2a8e4730;hb=8a063b3c5c7910db244bdab28e0409cde2ff551c;hpb=72db861c672a65ec0d2638b278421419e0fb0efd diff --git a/rarslave.py b/rarslave.py index 89d526d..a27c334 100644 --- a/rarslave.py +++ b/rarslave.py @@ -1,367 +1,240 @@ #!/usr/bin/env python # vim: set ts=4 sts=4 sw=4 textwidth=112 : -import re, os, sys -import par2parser +VERSION="2.0.0" +PROGRAM="rarslave2" -# Global Variables -(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4) +import os, sys, optparse, logging +import rsutil +import RarslaveDetector -class RarslaveExtractor (object): +# Global options from the rsutil.globals class +options = rsutil.globals.options +config = rsutil.globals.config - def __init__ (self, type): - self.type = type - self.heads = [] +# A tiny class to hold logging output until we're finished +class DelayedLogger (object): + def __init__ (self, output=sys.stdout.write): + self.__messages = [] + self.__output = output - def addHead (self, dir, head): - assert os.path.isdir (dir) - # REQUIRES that the dir is valid, but not that the file is valid, so that - # we can move a file that doesn't exist yet. - # FIXME: probably CAN add this back, since we should be running this AFTER repair. - #assert os.path.isfile (os.path.join (dir, head)) + def write (self, msg): + self.__messages.append (msg) - self.heads.append (os.path.join (dir, head)) + def flush (self): + pass - def extract (self, todir): - # Extract all heads of this set + def size (self): + """Returns the number of messages queued for printing""" + return len (self.__messages) - # Create the directory $todir if it doesn't exist - if not os.path.isdir (todir): - # TODO: LOGGER - try: - os.makedirs (todir) - except OSError: - # TODO: LOGGER - # Failed mkdir -p, clean up time ... - pass # FIXME: temporary for syntax + def close (self): + """Print all messages, clear the queue""" + for m in self.__messages: + self.__output (m) - # Extract all heads - extraction_func = \ - { TYPE_OLDRAR : self.__extract_rar, - TYPE_NEWRAR : self.__extract_rar, - TYPE_ZIP : self.__extract_zip, - TYPE_NOEXTRACT : self.__extract_noextract }[self.type] + self.__messages = [] - # Call the extraction function on each head - for h in self.heads: - extraction_func (h, todir) +# A tiny class used to find unique PAR2 sets +class CompareSet (object): - def __extract_rar (self, file, todir): - assert os.path.isfile (file) - assert os.path.isdir (todir) - - RAR_CMD = 'unrar x -o+ -- ' - - #file = full_abspath (file) - #todir = full_abspath (todir) + def __init__ (self, dir, p2file): + self.dir = dir + self.p2file = p2file - cmd = '%s \"%s\"' % (RAR_CMD, file) - ret = run_command (cmd, todir) + self.basename = rsutil.common.get_basename (self.p2file) + self.name_matches = rsutil.common.find_name_matches (self.dir, self.basename) - def __extract_zip (self, file, todir): - ZIP_CMD = 'unzip \"%s\" -d \"%s\"' + def __eq__ (self, rhs): + return (self.dir == rhs.dir) \ + and (self.basename == rhs.basename) \ + and rsutil.common.list_eq (self.name_matches, rhs.name_matches) - cmd = ZIP_CMD % (file, todir) - ret = run_command (cmd) - def __extract_noextract (self, file, todir): - # Just move this file to the $todir, since no extraction is needed - # FIXME: NOTE: mv will fail by itself if you're moving to the same dir! - cmd = 'mv \"%s\" \"%s\"' % (file, todir) - ret = run_command (cmd) +def find_all_par2_files (dir): + """Finds all par2 files in a directory""" + # NOTE: does NOT return absolute paths + if not os.path.isdir (os.path.abspath (dir)): + raise ValueError # bad directory given + dir = os.path.abspath (dir) + files = os.listdir (dir) -class RarslaveRepairer (object): - # Verify (and repair) the set - # Make sure it worked, otherwise clean up and return failure + return rsutil.common.find_par2_files (files) - def __init__ (self, dir, file, join=False): - self.dir = dir # the directory containing the par2 file - self.file = file # the par2 file - self.join = join # True if the par2 set is 001 002 ... +def generate_all_parsets (dir): + # Generate all parsets in the given directory. - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) + assert os.path.isdir (dir) # Directory MUST be valid - def checkAndRepair (self): - # Form the command: - # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES] - PAR2_CMD = 'par2repair -- ' + parsets = [] + p2files = find_all_par2_files (dir) - # Get set up - basename = get_basename (self.file) - all_files = find_likely_files (basename, self.dir) - all_files.sort () - par2_files = find_par2_files (all_files) + for f in p2files: + p = CompareSet (dir, f) + if p not in parsets: + parsets.append (p) - # assemble the command - command = "%s \"%s\" " % (PAR2_CMD, self.file) + return [(p.dir, p.p2file) for p in parsets] - for f in par2_files: - if f != self.file: - command += "\"%s\" " % get_filename(f) +def check_required_progs(): + """Check if the required programs are installed""" - if self.join: - for f in all_files: - if f not in par2_files: - command += "\"%s\" " % get_filename(f) + shell_not_found = 32512 + needed = [] - # run the command - ret = run_command (command, self.dir) + if rsutil.common.run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found: + needed.append ('par2repair') -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. + if rsutil.common.run_command ('unrar --help > /dev/null 2>&1') == shell_not_found: + needed.append ('unrar') - pwd = os.getcwd () + if rsutil.common.run_command ('unzip --help > /dev/null 2>&1') == shell_not_found: + needed.append ('unzip') - if indir != None: - assert os.path.isdir (indir) # MUST be a directory! - os.chdir (pwd) + if needed: + for n in needed: + print 'Needed program "%s" not found in $PATH' % (n, ) - # FIXME: re-enable this after testing - print 'RUNNING (%s): %s' % (indir, cmd) - # return os.system (cmd) + sys.exit(1) +def run_options (options): -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) + # Fix directories + options.work_dir = rsutil.common.full_abspath (options.work_dir) -def get_filename (f): - # TODO: I don't think that we should enforce this... - # TODO: ... because I think we should be able to get the filename, regardless - # TODO: of whether this is a legit filename RIGHT NOW or not. - # assert os.path.isfile (f) - return os.path.split (f)[1] + # Make sure that the directory is valid + if not os.path.isdir (options.work_dir): + sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir) + sys.stderr.write ('option to override the working directory temporarily, or edit the\n') + sys.stderr.write ('configuration file to override the working directory permanently.\n') + sys.exit (1) -def get_basename (name): - """Strips most kinds of endings from a filename""" + if options.extract_dir != None: + options.extract_dir = rsutil.common.full_abspath (options.extract_dir) - regex = '^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$' - r = re.compile (regex, re.IGNORECASE) - done = False + if options.version: + print PROGRAM + ' - ' + VERSION + print + print 'Copyright (c) 2005,2006 Ira W. Snyder (devel@irasnyder.com)' + print + print 'This program comes with ABSOLUTELY NO WARRANTY.' + print 'This is free software, and you are welcome to redistribute it' + print 'under certain conditions. See the file COPYING for details.' + sys.exit (0) - while not done: - done = True + if options.check_progs: + check_required_progs () - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False + if options.write_def_config: + config.write_config (default=True) - return name + if options.write_config: + config.write_config () -def find_likely_files (name, dir): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" +def find_loglevel (options): - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given + loglevel = options.verbose - options.quiet - dir = os.path.abspath (dir) - ename = re.escape (name) - regex = re.compile ('^%s.*$' % (ename, )) + if loglevel > 1: + loglevel = 1 - return [f for f in os.listdir (dir) if regex.match (f)] + if loglevel < -3: + loglevel = -3 -def find_par2_files (files): - """Find all par2 files in the list $files""" + LEVELS = { 1 : logging.DEBUG, + 0 : logging.INFO, + -1: logging.WARNING, + -2: logging.ERROR, + -3: logging.CRITICAL + } - regex = re.compile ('^.*\.par2$', re.IGNORECASE) - return [f for f in files if regex.match (f)] + return LEVELS [loglevel] -def find_all_par2_files (dir): - """Finds all par2 files in a directory""" - # NOTE: does NOT return absolute paths - - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given - - dir = os.path.abspath (dir) - files = os.listdir (dir) - - return find_par2_files (files) - -def has_extension (f, ext): - """Checks if f has the extension ext""" - - if ext[0] != '.': - ext = '.' + ext - - ext = re.escape (ext) - regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE) - return regex.match (f) - -def find_extraction_heads (dir, files): - """Takes a list of possible files and finds likely heads of - extraction.""" - - # NOTE: perhaps this should happen AFTER repair is - # NOTE: successful. That way all files would already exist - - # According to various sources online: - # 1) pre rar-3.0: .rar .r00 .r01 ... - # 2) post rar-3.0: .part01.rar .part02.rar - # 3) zip all ver: .zip - - extractor = None - p2files = find_par2_files (files) - - # Old RAR type, find all files ending in .rar - if is_oldrar (files): - extractor = RarslaveExtractor (TYPE_OLDRAR) - regex = re.compile ('^.*\.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_newrar (files): - extractor = RarslaveExtractor (TYPE_NEWRAR) - regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_zip (files): - extractor = RarslaveExtractor (TYPE_ZIP) - regex = re.compile ('^.*\.zip$', re.IGNORECASE) - for f in files: - if regex.match (f): - extractor.addHead (dir, f) - - if is_noextract (files): - # Use the Par2 Parser (from cfv) here to find out what files are protected. - # Since these are not being extracted, they will be mv'd to another directory - # later. - extractor = RarslaveExtractor (TYPE_NOEXTRACT) - - for f in p2files: - done = False - try: - prot_files = par2parser.get_protected_files (dir, f) - done = True - except: #FIXME: add the actual exceptions - print 'ERROR PARSING P2FILE ...', f - continue - - if done: - break - - if done: - for f in prot_files: - extractor.addHead (dir, f) - else: - print 'BADNESS' - - # Make sure we found the type - assert extractor != None - - return extractor - -def is_oldrar (files): - for f in files: - if has_extension (f, '.r00'): - return True - -def is_newrar (files): - for f in files: - if has_extension (f, '.part01.rar'): - return True - -def is_zip (files): - for f in files: - if has_extension (f, '.zip'): - return True - -def is_noextract (files): - # Type that needs no extraction. - # TODO: Add others ??? - for f in files: - if has_extension (f, '.001'): - return True - -def find_deleteable_files (files): - # Deleteable types regex should come from the config - dfiles = [] - dregex = re.compile ('^.*\.(par2|\d|\d\d\d|rar|r\d\d|zip)$', re.IGNORECASE) - - return [f for f in files if dregex.match (f)] - -def printlist (li): - for f in li: - print f - -class PAR2Set (object): - - dir = None - file = None - likely_files = [] - - def __init__ (self, dir, file): - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, file)) +def main (): - self.dir = dir - self.file = file + # Setup the logger + logger = DelayedLogger () + logging.basicConfig (stream=logger, level=logging.WARNING, \ + format='%(levelname)-8s %(message)s') - basename = get_basename (file) - self.likely_files = find_likely_files (basename, dir) + # Build the OptionParser + parser = optparse.OptionParser() + parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive', + default=rsutil.common.config_get_value('options', 'recursive'), + help="Don't run recursively") - def __list_eq (self, l1, l2): + parser.add_option('-d', '--work-dir', dest='work_dir', type='string', + default=rsutil.common.config_get_value('directories', 'working_directory'), + help="Start running at DIR", metavar='DIR') - if len(l1) != len(l2): - return False + parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string', + default=rsutil.common.config_get_value('directories', 'extract_directory'), + help="Extract to DIR", metavar='DIR') - for e in l1: - if e not in l2: - return False + parser.add_option('-p', '--check-required-programs', + action='store_true', dest='check_progs', + default=False, + help="Check for required programs") - return True + parser.add_option('-f', '--write-default-config', + action='store_true', dest='write_def_config', + default=False, help="Write out a new default config") - def __eq__ (self, rhs): - return self.__list_eq (self.likely_files, rhs.likely_files) + parser.add_option('-c', '--write-new-config', + action='store_true', dest='write_config', + default=False, help="Write out the current config") - def run_all (self): - par2files = find_par2_files (self.likely_files) - par2head = par2files[0] + parser.add_option('-i', '--interactive', dest='interactive', action='store_true', + default=rsutil.common.config_get_value('options', 'interactive'), + help="Confirm before removing files") - join = is_noextract (self.likely_files) + parser.add_option('-q', '--quiet', dest='quiet', action='count', + default=0, help="Output fatal messages only") - # Repair Stage - repairer = RarslaveRepairer (self.dir, par2head, join) - repairer.checkAndRepair () # FIXME: Check return value + parser.add_option('-v', '--verbose', dest='verbose', action='count', + default=0, help="Output extra information") - # Extraction Stage - extractor = find_extraction_heads (self.dir, self.likely_files) - extractor.extract ('extract_dir') # FIXME: Get it from the config + parser.add_option('-V', '--version', dest='version', action='store_true', + default=False, help="Output version information") - # Deletion Stage - printlist ( find_deleteable_files (self.likely_files) ) + parser.version = VERSION -def generate_all_parsets (dir): - # Generate all parsets in the given directory. + # Parse the given options + global options + (rsutil.globals.options, args) = parser.parse_args() + options = rsutil.globals.options - assert os.path.isdir (dir) # Directory MUST be valid + # Run any special actions that are needed on these options + run_options (options) - parsets = [] - p2files = find_all_par2_files (dir) + # Find the loglevel using the options given + logging.getLogger().setLevel (find_loglevel (options)) - for f in p2files: - p = PAR2Set (dir, f) - if p not in parsets: - parsets.append (p) + # Run recursively + if options.recursive: + for (dir, subdirs, files) in os.walk (options.work_dir): + parsets = generate_all_parsets (dir) + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () - return parsets + # Non-recursive + else: + parsets = generate_all_parsets (options.work_dir) + for (p2dir, p2file) in parsets: + detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) + ret = detector.runMatchingTypes () -def main (): - TOPDIR = os.path.abspath ('test_material') + # Print the results + if logger.size () > 0: + print '\nLog\n' + '=' * 80 + logger.close () - for (dir, subdirs, files) in os.walk (TOPDIR): - print 'DEBUG: IN DIRECTORY:', dir - parsets = generate_all_parsets (dir) - for p in parsets: - p.run_all () + # Done! + return 0 if __name__ == '__main__': main () +