#!/usr/bin/env python
# vim: set ts=4 sts=4 sw=4 textwidth=112 :
-import re, os, sys
+VERSION="2.0.0"
+PROGRAM="rarslave2"
-def get_basename (name):
- """Strips most kinds of endings from a filename"""
+import re, os, sys, optparse
+import Par2Parser
+import RarslaveConfig
+import RarslaveLogger
- regex = '^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$'
- r = re.compile (regex, re.IGNORECASE)
- done = False
+# Global Variables
+(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5)
+(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4)
+config = RarslaveConfig.RarslaveConfig()
+logger = RarslaveLogger.RarslaveLogger ()
- while not done:
- done = True
+# Global options to be set / used later.
+options = None
- if r.match (name):
- g = r.match (name).groups()
- name = g[0]
- done = False
+class RarslaveExtractor (object):
- return name
+ type = None
+ heads = []
-def find_likely_files (name, dir):
- """Finds files which are likely to be part of the set corresponding
- to $name in the directory $dir"""
+ def __init__ (self, dir, p2files, name_files, prot_files):
- if not os.path.isdir (os.path.abspath (dir)):
- raise ValueError # bad directory given
+ self.dir = dir
+ self.p2files = p2files
+ self.name_matched_files = name_files
+ self.prot_matched_files = prot_files
- dir = os.path.abspath (dir)
- ename = re.escape (name)
- regex = re.compile ('^%s.*$' % (ename, ))
+ # Find the type
+ self.type = self.__find_type ()
+
+ logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug)
+
+ # Find the heads
+ self.heads = self.__find_heads ()
+
+ for h in self.heads:
+ logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug)
+
+ def __repr__ (self):
+ return \
+ { TYPE_OLDRAR : 'Old RAR',
+ TYPE_NEWRAR : 'New RAR',
+ TYPE_ZIP : 'Zip',
+ TYPE_NOEXTRACT : 'No Extract',
+ TYPE_UNKNOWN : 'Unknown' } [self.type]
+
+ def __find_type (self):
+
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ if self.is_oldrar (all_files):
+ return TYPE_OLDRAR
+ elif self.is_newrar (all_files):
+ return TYPE_NEWRAR
+ elif self.is_zip (all_files):
+ return TYPE_ZIP
+ elif self.is_noextract (all_files):
+ return TYPE_NOEXTRACT
+
+ return TYPE_UNKNOWN
+
+ def __generic_find_heads (self, regex, ignorecase=True):
+
+ heads = []
+
+ if ignorecase:
+ cregex = re.compile (regex, re.IGNORECASE)
+ else:
+ cregex = re.compile (regex)
+
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ for f in all_files:
+ if cregex.match (f):
+ heads.append (f)
+
+ return heads
+
+ def __find_heads (self):
+
+ if self.type == TYPE_OLDRAR:
+ return self.__generic_find_heads ('^.*\.rar$')
+ elif self.type == TYPE_NEWRAR:
+ return self.__generic_find_heads ('^.*\.part0*1\.rar$')
+ elif self.type == TYPE_ZIP:
+ return self.__generic_find_heads ('^.*\.zip$')
+ elif self.type == TYPE_NOEXTRACT:
+ return self.prot_matched_files
+
+ return []
+
+ def __create_directory (self, dir):
+ if dir == None:
+ return SUCCESS
+
+ if os.path.isdir (dir):
+ return SUCCESS
+
+ try:
+ os.makedirs (dir)
+ logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose)
+ except OSError:
+ logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def runExtract (self, todir=None):
+ # Extract all heads of this set
+
+ # Extract to the head's dir if we don't care where to extract
+ if todir == None:
+ todir = self.dir
+
+ # Create the directory $todir if it doesn't exist
+ ret = self.__create_directory (todir)
+
+ if ret != SUCCESS:
+ return -EEXTRACT
+
+ # Extract all heads
+ extraction_func = \
+ { TYPE_OLDRAR : self.__extract_rar,
+ TYPE_NEWRAR : self.__extract_rar,
+ TYPE_ZIP : self.__extract_zip,
+ TYPE_NOEXTRACT : self.__extract_noextract,
+ TYPE_UNKNOWN : self.__extract_unknown }[self.type]
+
+ # Call the extraction function on each head
+ for h in self.heads:
+ full_head = full_abspath (h)
+ ret = extraction_func (full_head, todir)
+ logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug)
+
+ # Check error code
+ if ret != SUCCESS:
+ logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def __extract_rar (self, file, todir):
+ assert os.path.isfile (file)
+ assert os.path.isdir (todir)
+
+ RAR_CMD = config.get_value ('commands', 'unrar')
+
+ cmd = '%s \"%s\"' % (RAR_CMD, file)
+ ret = run_command (cmd, todir)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def __extract_zip (self, file, todir):
+ ZIP_CMD = config.get_value ('commands', 'unzip')
+
+ cmd = ZIP_CMD % (file, todir)
+ ret = run_command (cmd)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def __extract_noextract (self, file, todir):
+ # Just move this file to the $todir, since no extraction is needed
+ # FIXME: NOTE: mv will fail by itself if you're moving to the same dir!
+ NOEXTRACT_CMD = config.get_value ('commands', 'noextract')
+
+ # Make sure that both files are not the same file. If they are, don't run at all.
+ if os.path.samefile (file, os.path.join (todir, file)):
+ return SUCCESS
+
+ cmd = NOEXTRACT_CMD % (file, todir)
+ ret = run_command (cmd)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def __extract_unknown (self, file, todir):
+ return SUCCESS
+
+ def __generic_matcher (self, files, regex, nocase=False):
+ """Run the regex over the files, and see if one matches or not.
+ NOTE: this does not return the matches, just if a match occurred."""
+
+ if nocase:
+ cregex = re.compile (regex, re.IGNORECASE)
+ else:
+ cregex = re.compile (regex)
+
+ for f in files:
+ if cregex.match (f):
+ return True
+
+ return False
+
+ def is_oldrar (self, files):
+ return self.__generic_matcher (files, '^.*\.r00$')
+
+ def is_newrar (self, files):
+ return self.__generic_matcher (files, '^.*\.part0*1\.rar$')
+
+ def is_zip (self, files):
+ return self.__generic_matcher (files, '^.*\.zip$')
+
+ def is_noextract (self, files):
+ # Type that needs no extraction.
+ # TODO: Add others ???
+ return self.__generic_matcher (files, '^.*\.001$')
+
+class PAR2Set (object):
+
+ dir = None
+ p2file = None # The starting par2
+ basename = None # The p2file's basename
+ all_p2files = []
+ name_matched_files = [] # Files that match by basename of the p2file
+ prot_matched_files = [] # Files that match by being protected members
+
+ def __init__ (self, dir, p2file):
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, p2file))
+
+ self.dir = dir
+ self.p2file = p2file
+ self.basename = self.__get_basename (p2file)
+
+ # Find files that match by name only
+ self.name_matched_files = self.__find_name_matches (self.dir, self.basename)
+
+ # Find all par2 files for this set using name matches
+ self.all_p2files = find_par2_files (self.name_matched_files)
+
+ # Try to get the protected files for this set
+ self.prot_matched_files = self.__parse_all_par2 ()
+
+ def __list_eq (self, l1, l2):
+
+ if len(l1) != len(l2):
+ return False
+
+ for e in l1:
+ if e not in l2:
+ return False
+
+ return True
+
+ def __eq__ (self, rhs):
+ return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \
+ self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \
+ self.__list_eq (self.prot_matched_files, rhs.prot_matched_files)
+
+ def __get_basename (self, name):
+ """Strips most kinds of endings from a filename"""
+
+ regex = config.get_value ('regular expressions', 'basename_regex')
+ r = re.compile (regex, re.IGNORECASE)
+ done = False
+
+ while not done:
+ done = True
+
+ if r.match (name):
+ g = r.match (name).groups()
+ name = g[0]
+ done = False
+
+ return name
+
+ def __parse_all_par2 (self):
+ """Searches though self.all_p2files and tries to parse at least one of them"""
+ done = False
+ files = []
+
+ for f in self.all_p2files:
+
+ # Exit early if we've found a good file
+ if done:
+ break
+
+ try:
+ files = Par2Parser.get_protected_files (self.dir, f)
+ done = True
+ except (EnvironmentError, OSError, OverflowError):
+ logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal)
+
+ # Now that we're out of the loop, check if we really finished
+ if not done:
+ logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+
+ # Return whatever we've got, empty or not
+ return files
+
+ def __find_name_matches (self, dir, basename):
+ """Finds files which are likely to be part of the set corresponding
+ to $name in the directory $dir"""
+
+ assert os.path.isdir (dir)
+
+ ename = re.escape (basename)
+ regex = re.compile ('^%s.*$' % (ename, ))
- return [f for f in os.listdir (dir) if regex.match (f)]
+ return [f for f in os.listdir (dir) if regex.match (f)]
+
+ def __update_name_matches (self):
+ """Updates the self.name_matched_files variable with the most current information.
+ This should be called after the directory contents are likely to change."""
+
+ self.name_matched_files = self.__find_name_matches (self.dir, self.basename)
+
+ def __is_joinfile (self, filename):
+ regex = re.compile ('^.*\.\d\d\d$', re.IGNORECASE)
+ if regex.match (filename):
+ return True
+
+ return False
+
+ def __should_be_joined (self, files):
+ for f in files:
+ if self.__is_joinfile (f):
+ return True
+
+ def runCheckAndRepair (self):
+ PAR2_CMD = config.get_value ('commands', 'par2repair')
+
+ # Get set up
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+ join = self.__should_be_joined (all_files)
+
+ # assemble the command
+ # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES]
+ command = "%s \"%s\" " % (PAR2_CMD, self.p2file)
+
+ for f in self.all_p2files:
+ if f != self.p2file:
+ command += "\"%s\" " % os.path.split (f)[1]
+
+ # Only needed when using par2 to join
+ if join:
+ for f in all_files:
+ if self.__is_joinfile (f):
+ command += "\"%s\" " % os.path.split (f)[1]
+
+ # run the command
+ ret = run_command (command, self.dir)
+
+ # check the result
+ if ret != 0:
+ logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+ return -ECHECK
+
+ return SUCCESS
+
+ def __find_deleteable_files (self):
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+ DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex')
+ dregex = re.compile (DELETE_REGEX, re.IGNORECASE)
+
+ return [f for f in all_files if dregex.match (f)]
+
+ def __delete_list_of_files (self, dir, files, interactive=False):
+ # Delete a list of files
+
+ assert os.path.isdir (dir)
+
+ done = False
+ valid_y = ['Y', 'YES']
+ valid_n = ['N', 'NO', '']
+
+ if interactive:
+ while not done:
+ print 'Do you want to delete the following?:'
+ for f in files:
+ print f
+ s = raw_input ('Delete [y/N]: ').upper()
+
+ if s in valid_y + valid_n:
+ done = True
+
+ if s in valid_n:
+ return SUCCESS
+
+ for f in files:
+ try:
+ os.remove (os.path.join (dir, f))
+ logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug)
+ except:
+ logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f),
+ RarslaveLogger.MessageType.Fatal)
+ return -EDELETE
+
+ return SUCCESS
+
+ def runDelete (self):
+ deleteable_files = self.__find_deleteable_files ()
+ ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive)
+
+ return ret
+
+ def run_all (self):
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ # Repair Stage
+ ret = self.runCheckAndRepair ()
+
+ if ret != SUCCESS:
+ logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+ return -ECHECK
+
+ self.__update_name_matches ()
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ # Extraction Stage
+ extractor = RarslaveExtractor (self.dir, self.all_p2files, \
+ self.name_matched_files, self.prot_matched_files)
+ ret = extractor.runExtract (options.extract_dir)
+
+ if ret != SUCCESS:
+ logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ self.__update_name_matches ()
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ # Deletion Stage
+ ret = self.runDelete ()
+
+ if ret != SUCCESS:
+ logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+ return -EDELETE
+
+ logger.addMessage ('Successfully completed: %s' % self.p2file)
+ return SUCCESS
+
+def run_command (cmd, indir=None):
+ # Runs the specified command-line in the directory given (or, in the current directory
+ # if none is given). It returns the status code given by the application.
+
+ pwd = os.getcwd ()
+
+ if indir != None:
+ assert os.path.isdir (indir) # MUST be a directory!
+ os.chdir (indir)
+
+ print 'RUNNING (%s): %s' % (indir, cmd)
+ ret = os.system (cmd)
+ os.chdir (pwd)
+ return ret
+
+def full_abspath (p):
+ return os.path.abspath (os.path.expanduser (p))
+
+def find_par2_files (files):
+ """Find all par2 files in the list $files"""
+
+ PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex')
+ regex = re.compile (PAR2_REGEX, re.IGNORECASE)
+ return [f for f in files if regex.match (f)]
def find_all_par2_files (dir):
"""Finds all par2 files in a directory"""
+ # NOTE: does NOT return absolute paths
if not os.path.isdir (os.path.abspath (dir)):
raise ValueError # bad directory given
dir = os.path.abspath (dir)
- regex = re.compile ('^.*\.par2$', re.IGNORECASE)
+ files = os.listdir (dir)
- # Find all files
- return [f for f in os.listdir (dir) if regex.match (f)]
+ return find_par2_files (files)
-def has_extension (f, ext):
- """Checks if f has the extension ext"""
+def no_duplicates (li):
+ """Removes all duplicates from a list"""
+ return list(set(li))
- if ext[0] != '.':
- ext = '.' + ext
+def generate_all_parsets (dir):
+ # Generate all parsets in the given directory.
- ext = re.escape (ext)
- regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE)
- return regex.match (f)
+ assert os.path.isdir (dir) # Directory MUST be valid
-def find_extraction_heads (files):
- """Takes a list of possible files and finds likely heads of
- extraction."""
+ parsets = []
+ p2files = find_all_par2_files (dir)
- # NOTE: perhaps this should happen AFTER repair is
- # NOTE: successful. That way all files would already exist
+ for f in p2files:
+ p = PAR2Set (dir, f)
+ if p not in parsets:
+ parsets.append (p)
- # According to various sources online:
- # 1) pre rar-3.0: .rar .r00 .r01 ...
- # 2) post rar-3.0: .part01.rar .part02.rar
- # 3) zip all ver: .zip
+ return parsets
- heads = []
+def check_required_progs():
+ """Check if the required programs are installed"""
- # Old RAR type, find all files ending in .rar
- if is_oldrar (files):
- regex = re.compile ('^.*\.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- heads.append (f)
+ shell_not_found = 32512
+ needed = []
- return heads
+ if run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('par2repair')
- if is_newrar (files):
- regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- heads.append (f)
+ if run_command ('unrar --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('unrar')
- return heads
+ if run_command ('unzip --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('unzip')
- if is_zip (files):
- regex = re.compile ('^.*\.zip$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- heads.append (f)
+ if needed:
+ for n in needed:
+ print 'Needed program "%s" not found in $PATH' % (n, )
- return heads
+ sys.exit(1)
- # Not a type we know yet
- raise ValueError
+def run_options (options):
+ # Fix directories
+ options.work_dir = full_abspath (options.work_dir)
-def is_oldrar (files):
- for f in files:
- if has_extension (f, '.r00'):
- return True
+ # Make sure that the directory is valid
+ if not os.path.isdir (options.work_dir):
+ sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir)
+ sys.stderr.write ('option to override the working directory temporarily, or edit the\n')
+ sys.stderr.write ('configuration file to override the working directory permanently.\n')
+ sys.exit (1)
-def is_newrar (files):
- for f in files:
- if has_extension (f, '.part01.rar'):
- return True
+ if options.extract_dir != None:
+ options.extract_dir = full_abspath (options.extract_dir)
-def is_zip (files):
- for f in files:
- if has_extension (f, '.zip'):
- return True
+ if options.version:
+ print PROGRAM + ' - ' + VERSION
+ print
+ print 'Copyright (c) 2005,2006 Ira W. Snyder (devel@irasnyder.com)'
+ print
+ print 'This program comes with ABSOLUTELY NO WARRANTY.'
+ print 'This is free software, and you are welcome to redistribute it'
+ print 'under certain conditions. See the file COPYING for details.'
+ sys.exit (0)
-def is_noextract (files):
- # Type that needs no extraction.
- # TODO: Add others ???
- for f in files:
- if has_extension (f, '.001'):
- return True
+ if options.check_progs:
+ check_required_progs ()
+
+ if options.write_def_config:
+ config.write_config (default=True)
+
+ if options.write_config:
+ config.write_config ()
+
+def find_loglevel (options):
+
+ loglevel = options.verbose - options.quiet
+
+ if loglevel < RarslaveLogger.MessageType.Fatal:
+ loglevel = RarslaveLogger.MessageType.Fatal
-def find_deleteable_files (files):
- # Deleteable types regex should come from the config
- dfiles = []
- dregex = re.compile ('^.*\.(par2|\d|\d\d\d|rar|r\d\d|zip)$', re.IGNORECASE)
+ if loglevel > RarslaveLogger.MessageType.Debug:
+ loglevel = RarslaveLogger.MessageType.Debug
- return [f for f in files if dregex.match (f)]
+ return loglevel
-def extract (heads, todir):
- # Try to extract each head
- # NOTE: REQUIRES full paths to heads
+def printMessageTable (loglevel):
- PWD = os.getcwd()
+ if logger.hasFatalMessages ():
+ print '\nFatal Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Fatal)
- # FIXME: Should come from the config
- RCMD = 'unrar x -o+ -- '
- ZCMD = 'unzip -- '
+ if loglevel == RarslaveLogger.MessageType.Fatal:
+ return
- for h in heads:
- # find type
- # extract it
+ if logger.hasNormalMessages ():
+ print '\nNormal Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Normal)
- # NOTE: probably not able to clean up effectively...
- pass
+ if loglevel == RarslaveLogger.MessageType.Normal:
+ return
+ if logger.hasVerboseMessages ():
+ print '\nVerbose Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Verbose)
+ if loglevel == RarslaveLogger.MessageType.Verbose:
+ return
+
+ if logger.hasDebugMessages ():
+ print '\nDebug Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Debug)
+
+ return
def main ():
- print find_all_par2_files ('/home/irasnyd/downloads/test_material/01/')
+
+ # Build the OptionParser
+ parser = optparse.OptionParser()
+ parser.add_option('-n', '--not-recursive',
+ action='store_false', dest='recursive',
+ default=config.get_value('options', 'recursive'),
+ help="Don't run recursively")
+
+ parser.add_option('-d', '--work-dir',
+ dest='work_dir', type='string',
+ default=config.get_value('directories', 'working_directory'),
+ help="Start running at DIR", metavar='DIR')
+
+ parser.add_option('-e', '--extract-dir',
+ dest='extract_dir', type='string',
+ default=config.get_value('directories', 'extract_directory'),
+ help="Extract to DIR", metavar='DIR')
+
+ parser.add_option('-p', '--check-required-programs',
+ action='store_true', dest='check_progs',
+ default=False,
+ help="Check for required programs")
+
+ parser.add_option('-f', '--write-default-config',
+ action='store_true', dest='write_def_config',
+ default=False, help="Write out a new default config")
+
+ parser.add_option('-c', '--write-new-config',
+ action='store_true', dest='write_config',
+ default=False, help="Write out the current config")
+
+ parser.add_option('-i', '--interactive', dest='interactive', action='store_true',
+ default=config.get_value('options', 'interactive'),
+ help="Confirm before removing files")
+
+ parser.add_option('-q', '--quiet', dest='quiet', action='count',
+ default=0, help="Output fatal messages only")
+
+ parser.add_option('-v', '--verbose', dest='verbose', action='count',
+ default=0, help="Output extra information")
+
+ parser.add_option('-V', '--version', dest='version', action='store_true',
+ default=False, help="Output version information")
+
+ parser.version = VERSION
+
+ # Parse the given options
+ global options
+ (options, args) = parser.parse_args()
+
+ # Run any special actions that are needed on these options
+ run_options (options)
+
+ # Find the loglevel using the options given
+ loglevel = find_loglevel (options)
+
+ # Run recursively
+ if options.recursive:
+ for (dir, subdirs, files) in os.walk (options.work_dir):
+ parsets = generate_all_parsets (dir)
+ for p in parsets:
+ p.run_all ()
+
+ # Non-recursive
+ else:
+ parsets = generate_all_parsets (options.work_dir)
+ for p in parsets:
+ p.run_all ()
+
+ # Print the results
+ printMessageTable (loglevel)
+
+ # Done!
+ return 0
if __name__ == '__main__':
main ()
+
projects / rarslave2.git / blobdiff