#!/usr/bin/env python
# vim: set ts=4 sts=4 sw=4 textwidth=112 :
-import re, os, sys
+VERSION="2.0.0"
+PROGRAM="rarslave2"
+
+import re, os, sys, optparse
+import par2parser
+import RarslaveConfig
+import RarslaveLogger
# Global Variables
(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4)
+(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4)
+config = RarslaveConfig.RarslaveConfig()
+logger = RarslaveLogger.RarslaveLogger ()
+
+# Global options to be set / used later.
+options = None
class RarslaveExtractor (object):
self.type = type
self.heads = []
- def addHead (self, head):
- assert not os.path.isfile (head)
+ def addHead (self, dir, head):
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, head))
- self.heads.append (head)
+ full_head = os.path.join (dir, head)
+ logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug)
+ self.heads.append (full_head)
- def extract (self, todir):
- # Extract all heads
+ def extract (self, todir=None):
+ # Extract all heads of this set
+ # Create the directory $todir if it doesn't exist
+ if todir != None and not os.path.isdir (todir):
+ logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose)
+ try:
+ os.makedirs (todir)
+ except OSError:
+ logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ # Extract all heads
extraction_func = \
{ TYPE_OLDRAR : self.__extract_rar,
TYPE_NEWRAR : self.__extract_rar,
# Call the extraction function on each head
for h in self.heads:
- extraction_func (h, todir)
+ if todir == None:
+ # Run in the head's directory
+ ret = extraction_func (h, os.path.dirname (h))
+ else:
+ ret = extraction_func (h, todir)
+
+ logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug)
+
+ # Check error code
+ if ret != SUCCESS:
+ logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ return SUCCESS
def __extract_rar (self, file, todir):
- print 'Extracting (%s, %s)' % (file, todir)
+ assert os.path.isfile (file)
+ assert os.path.isdir (todir)
+
+ RAR_CMD = config.get_value ('commands', 'unrar')
+
+ cmd = '%s \"%s\"' % (RAR_CMD, file)
+ ret = run_command (cmd, todir)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
def __extract_zip (self, file, todir):
- print 'Extracting (%s, %s)' % (file, todir)
+ ZIP_CMD = config.get_value ('commands', 'unzip')
+
+ cmd = ZIP_CMD % (file, todir)
+ ret = run_command (cmd)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
def __extract_noextract (self, file, todir):
- print 'Extracting (%s, %s)' % (file, todir)
+ # Just move this file to the $todir, since no extraction is needed
+ # FIXME: NOTE: mv will fail by itself if you're moving to the same dir!
+ NOEXTRACT_CMD = config.get_value ('commands', 'noextract')
+
+ # Make sure that both files are not the same file. If they are, don't run at all.
+ if os.path.samefile (file, os.path.join (todir, file)):
+ return SUCCESS
+
+ cmd = NOEXTRACT_CMD % (file, todir)
+ ret = run_command (cmd)
+
+ # Check error code
+ if ret != 0:
+ return -EEXTRACT
+
+ return SUCCESS
+
+
+
+class RarslaveRepairer (object):
+ # Verify (and repair) the set
+ # Make sure it worked, otherwise clean up and return failure
+
+ def __init__ (self, dir, file, join=False):
+ self.dir = dir # the directory containing the par2 file
+ self.file = file # the par2 file
+ self.join = join # True if the par2 set is 001 002 ...
+
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, file))
+
+ def checkAndRepair (self):
+ # Form the command:
+ # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES]
+ PAR2_CMD = config.get_value ('commands', 'par2repair')
+
+ # Get set up
+ basename = get_basename (self.file)
+ all_files = find_likely_files (self.dir, self.file)
+ all_files.sort ()
+ par2_files = find_par2_files (all_files)
+
+ # assemble the command
+ command = "%s \"%s\" " % (PAR2_CMD, self.file)
+
+ for f in par2_files:
+ if f != self.file:
+ command += "\"%s\" " % os.path.split (f)[1]
+
+ if self.join:
+ for f in all_files:
+ if f not in par2_files:
+ command += "\"%s\" " % os.path.split (f)[1]
+
+ # run the command
+ ret = run_command (command, self.dir)
+
+ # check the result
+ if ret != 0:
+ logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal)
+ return -ECHECK
+
+ return SUCCESS
+
+def run_command (cmd, indir=None):
+ # Runs the specified command-line in the directory given (or, in the current directory
+ # if none is given). It returns the status code given by the application.
+
+ pwd = os.getcwd ()
+
+ if indir != None:
+ assert os.path.isdir (indir) # MUST be a directory!
+ os.chdir (indir)
+
+ ret = os.system (cmd)
+ os.chdir (pwd)
+ return ret
+def full_abspath (p):
+ return os.path.abspath (os.path.expanduser (p))
def get_basename (name):
"""Strips most kinds of endings from a filename"""
- regex = '^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$'
+ regex = config.get_value ('regular expressions', 'basename_regex')
r = re.compile (regex, re.IGNORECASE)
done = False
return name
-def find_likely_files (name, dir):
+def find_likely_files (dir, p2file):
"""Finds files which are likely to be part of the set corresponding
to $name in the directory $dir"""
- if not os.path.isdir (os.path.abspath (dir)):
- raise ValueError # bad directory given
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, p2file))
+
+ basename = get_basename (p2file)
dir = os.path.abspath (dir)
- ename = re.escape (name)
+ ename = re.escape (basename)
regex = re.compile ('^%s.*$' % (ename, ))
- return [os.path.abspath(f) for f in os.listdir (dir) if regex.match (f)]
+ name_matches = [f for f in os.listdir (dir) if regex.match (f)]
+ try:
+ parsed_matches = par2parser.get_protected_files (dir, p2file)
+ except EnvironmentError:
+ parsed_matches = []
+ logger.addMessage ('Bad par2 file: %s' % p2file, RarslaveLogger.MessageType.Fatal)
+
+ return name_matches + parsed_matches
+
+def find_par2_files (files):
+ """Find all par2 files in the list $files"""
+
+ PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex')
+ regex = re.compile (PAR2_REGEX, re.IGNORECASE)
+ return [f for f in files if regex.match (f)]
def find_all_par2_files (dir):
"""Finds all par2 files in a directory"""
+ # NOTE: does NOT return absolute paths
if not os.path.isdir (os.path.abspath (dir)):
raise ValueError # bad directory given
dir = os.path.abspath (dir)
- regex = re.compile ('^.*\.par2$', re.IGNORECASE)
-
- # Find all files
- return [f for f in os.listdir (dir) if regex.match (f)]
-
-def has_extension (f, ext):
- """Checks if f has the extension ext"""
-
- if ext[0] != '.':
- ext = '.' + ext
+ files = os.listdir (dir)
- ext = re.escape (ext)
- regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE)
- return regex.match (f)
+ return find_par2_files (files)
-def find_extraction_heads (files):
+def find_extraction_heads (dir, files):
"""Takes a list of possible files and finds likely heads of
extraction."""
# 3) zip all ver: .zip
extractor = None
+ p2files = find_par2_files (files)
# Old RAR type, find all files ending in .rar
if is_oldrar (files):
regex = re.compile ('^.*\.rar$', re.IGNORECASE)
for f in files:
if regex.match (f):
- extractor.addHead (f)
+ extractor.addHead (dir, f)
if is_newrar (files):
extractor = RarslaveExtractor (TYPE_NEWRAR)
- regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE)
+ regex = re.compile ('^.*\.part0*1.rar$', re.IGNORECASE)
for f in files:
if regex.match (f):
- extractor.addHead (f)
+ extractor.addHead (dir, f)
if is_zip (files):
extractor = RarslaveExtractor (TYPE_ZIP)
regex = re.compile ('^.*\.zip$', re.IGNORECASE)
for f in files:
if regex.match (f):
- extractor.addHead (f)
+ extractor.addHead (dir, f)
if is_noextract (files):
+ # Use the Par2 Parser (from cfv) here to find out what files are protected.
+ # Since these are not being extracted, they will be mv'd to another directory
+ # later.
extractor = RarslaveExtractor (TYPE_NOEXTRACT)
+ for f in p2files:
+ done = False
+ try:
+ prot_files = par2parser.get_protected_files (dir, f)
+ done = True
+ except EnvironmentError:
+ logger.addMessage ('Error parsing PAR2 file: %s', f)
+ continue
+
+ if done:
+ break
+
+ if done:
+ for f in prot_files:
+ extractor.addHead (dir, f)
+ else:
+ logger.addMessage ('Error parsing all PAR2 files in this set ...')
+
# Make sure we found the type
- assert extractor != None
+ if extractor == None:
+ logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0],
+ RarslaveLogger.MessageType.Verbose)
+
+ # No-heads here, but it's better than failing completely
+ extractor = RarslaveExtractor (TYPE_NOEXTRACT)
return extractor
-def is_oldrar (files):
+def generic_matcher (files, regex, nocase=False):
+ """Run the regex over the files, and see if one matches or not.
+ NOTE: this does not return the matches, just if a match occurred."""
+
+ if nocase:
+ cregex = re.compile (regex, re.IGNORECASE)
+ else:
+ cregex = re.compile (regex)
+
for f in files:
- if has_extension (f, '.r00'):
+ if cregex.match (f):
return True
+ return False
+
+def is_oldrar (files):
+ return generic_matcher (files, '^.*\.r00$')
+
def is_newrar (files):
- for f in files:
- if has_extension (f, '.part01.rar'):
- return True
+ return generic_matcher (files, '^.*\.part0*1\.rar$')
def is_zip (files):
- for f in files:
- if has_extension (f, '.zip'):
- return True
+ return generic_matcher (files, '^.*\.zip$')
def is_noextract (files):
# Type that needs no extraction.
# TODO: Add others ???
- for f in files:
- if has_extension (f, '.001'):
- return True
+ return generic_matcher (files, '^.*\.001$')
def find_deleteable_files (files):
# Deleteable types regex should come from the config
dfiles = []
- dregex = re.compile ('^.*\.(par2|\d|\d\d\d|rar|r\d\d|zip)$', re.IGNORECASE)
+ DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex')
+ dregex = re.compile (DELETE_REGEX, re.IGNORECASE)
return [f for f in files if dregex.match (f)]
for f in li:
print f
+class PAR2Set (object):
+
+ dir = None
+ file = None
+ likely_files = []
+
+ def __init__ (self, dir, file):
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, file))
+
+ self.dir = dir
+ self.file = file
+
+ basename = get_basename (file)
+ self.likely_files = find_likely_files (dir, file)
+
+ def __list_eq (self, l1, l2):
+
+ if len(l1) != len(l2):
+ return False
+
+ for e in l1:
+ if e not in l2:
+ return False
+
+ return True
+
+ def __eq__ (self, rhs):
+ return self.__list_eq (self.likely_files, rhs.likely_files)
+
+ def run_all (self):
+ par2files = find_par2_files (self.likely_files)
+ par2head = par2files[0]
+
+ join = is_noextract (self.likely_files)
+
+ # Repair Stage
+ repairer = RarslaveRepairer (self.dir, par2head, join)
+ ret = repairer.checkAndRepair ()
+
+ if ret != SUCCESS:
+ logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ return -ECHECK
+
+ # Extraction Stage
+ EXTRACT_DIR = options.extract_dir
+ extractor = find_extraction_heads (self.dir, self.likely_files)
+ ret = extractor.extract (EXTRACT_DIR)
+
+ if ret != SUCCESS:
+ logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ # Deletion Stage
+ DELETE_INTERACTIVE = options.interactive
+ deleteable_files = find_deleteable_files (self.likely_files)
+ ret = delete_list (self.dir, deleteable_files, DELETE_INTERACTIVE)
+
+ if ret != SUCCESS:
+ logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ return -EDELETE
+
+ logger.addMessage ('Successfully completed: %s' % par2head)
+ return SUCCESS
+
+def delete_list (dir, files, interactive=False):
+ # Delete a list of files
+
+ assert os.path.isdir (dir)
+
+ done = False
+ valid_y = ['Y', 'YES']
+ valid_n = ['N', 'NO']
+
+ if interactive:
+ while not done:
+ print 'Do you want to delete the following?:'
+ printlist (files)
+ s = raw_input ('Delete [y/N]: ').upper()
+
+ if s in valid_y + valid_n:
+ done = True
+
+ if s in valid_n:
+ return SUCCESS
+
+ for f in files:
+ os.remove (os.path.join (dir, f))
+
+ return SUCCESS
+
+
+def generate_all_parsets (dir):
+ # Generate all parsets in the given directory.
+
+ assert os.path.isdir (dir) # Directory MUST be valid
+
+ parsets = []
+ p2files = find_all_par2_files (dir)
+
+ for f in p2files:
+ p = PAR2Set (dir, f)
+ if p not in parsets:
+ parsets.append (p)
+
+ return parsets
+
+def check_required_progs():
+ """Check if the required programs are installed"""
+
+ shell_not_found = 32512
+ needed = []
+
+ if run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('par2repair')
+
+ if run_command ('unrar --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('unrar')
+
+ if run_command ('unzip --help > /dev/null 2>&1') == shell_not_found:
+ needed.append ('unzip')
+
+ if needed:
+ for n in needed:
+ print 'Needed program "%s" not found in $PATH' % (n, )
+
+ sys.exit(1)
+
+def run_options (options):
+
+ # Fix directories
+ options.work_dir = full_abspath (options.work_dir)
+
+ # Make sure that the directory is valid
+ if not os.path.isdir (options.work_dir):
+ sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir)
+ sys.stderr.write ('option to override the working directory temporarily, or edit the\n')
+ sys.stderr.write ('configuration file to override the working directory permanently.\n')
+ sys.exit (1)
+
+ if options.extract_dir != None:
+ options.extract_dir = full_abspath (options.extract_dir)
+
+ if options.version:
+ print PROGRAM + ' - ' + VERSION
+ print
+ print 'Copyright (c) 2005,2006 Ira W. Snyder (devel@irasnyder.com)'
+ print
+ print 'This program comes with ABSOLUTELY NO WARRANTY.'
+ print 'This is free software, and you are welcome to redistribute it'
+ print 'under certain conditions. See the file COPYING for details.'
+ sys.exit (0)
+
+ if options.check_progs:
+ check_required_progs ()
+
+ if options.write_def_config:
+ config.write_config (default=True)
+
+ if options.write_config:
+ config.write_config ()
+
+def find_loglevel (options):
+
+ loglevel = options.verbose - options.quiet
+
+ if loglevel < RarslaveLogger.MessageType.Fatal:
+ loglevel = RarslaveLogger.MessageType.Fatal
+
+ if loglevel > RarslaveLogger.MessageType.Debug:
+ loglevel = RarslaveLogger.MessageType.Debug
+
+ return loglevel
+
+def printMessageTable (loglevel):
+
+ if logger.hasFatalMessages ():
+ print '\nFatal Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Fatal)
+
+ if loglevel == RarslaveLogger.MessageType.Fatal:
+ return
+
+ if logger.hasNormalMessages ():
+ print '\nNormal Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Normal)
+
+ if loglevel == RarslaveLogger.MessageType.Normal:
+ return
+
+ if logger.hasVerboseMessages ():
+ print '\nVerbose Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Verbose)
+
+ if loglevel == RarslaveLogger.MessageType.Verbose:
+ return
+
+ if logger.hasDebugMessages ():
+ print '\nDebug Messages\n' + '=' * 80
+ logger.printLoglevel (RarslaveLogger.MessageType.Debug)
+
+ return
+
def main ():
- DIR = os.path.abspath ('test_material/01/')
- p2files = find_all_par2_files (DIR)
- files = find_likely_files (get_basename (p2files[0]), DIR)
- find_extraction_heads (files).extract('nodir')
- print 'DELETEABLE_FILES:'
- printlist ( find_deleteable_files (files) )
-
- print
-
- DIR = os.path.abspath ('test_material/13/')
- p2files = find_all_par2_files (DIR)
- files = find_likely_files (get_basename (p2files[0]), DIR)
- find_extraction_heads (files).extract ('nodir')
- print 'DELETEABLE_FILES:'
- printlist ( find_deleteable_files (files) )
+
+ # Build the OptionParser
+ parser = optparse.OptionParser()
+ parser.add_option('-n', '--not-recursive',
+ action='store_false', dest='recursive',
+ default=config.get_value('options', 'recursive'),
+ help="Don't run recursively")
+
+ parser.add_option('-d', '--work-dir',
+ dest='work_dir', type='string',
+ default=config.get_value('directories', 'working_directory'),
+ help="Start running at DIR", metavar='DIR')
+
+ parser.add_option('-e', '--extract-dir',
+ dest='extract_dir', type='string',
+ default=config.get_value('directories', 'extract_directory'),
+ help="Extract to DIR", metavar='DIR')
+
+ parser.add_option('-p', '--check-required-programs',
+ action='store_true', dest='check_progs',
+ default=False,
+ help="Check for required programs")
+
+ parser.add_option('-f', '--write-default-config',
+ action='store_true', dest='write_def_config',
+ default=False, help="Write out a new default config")
+
+ parser.add_option('-c', '--write-new-config',
+ action='store_true', dest='write_config',
+ default=False, help="Write out the current config")
+
+ parser.add_option('-i', '--interactive', dest='interactive', action='store_true',
+ default=config.get_value('options', 'interactive'),
+ help="Confirm before removing files")
+
+ parser.add_option('-q', '--quiet', dest='quiet', action='count',
+ default=0, help="Output fatal messages only")
+
+ parser.add_option('-v', '--verbose', dest='verbose', action='count',
+ default=0, help="Output extra information")
+
+ parser.add_option('-V', '--version', dest='version', action='store_true',
+ default=False, help="Output version information")
+
+ parser.version = VERSION
+
+ # Parse the given options
+ global options
+ (options, args) = parser.parse_args()
+
+ # Run any special actions that are needed on these options
+ run_options (options)
+
+ # Find the loglevel using the options given
+ loglevel = find_loglevel (options)
+
+ # Run recursively
+ if options.recursive:
+ for (dir, subdirs, files) in os.walk (options.work_dir):
+ parsets = generate_all_parsets (dir)
+ for p in parsets:
+ p.run_all ()
+
+ # Non-recursive
+ else:
+ parsets = generate_all_parsets (options.work_dir)
+ for p in parsets:
+ p.run_all ()
+
+ # Print the results
+ printMessageTable (loglevel)
+
+ # Done!
+ return 0
if __name__ == '__main__':
main ()
+
projects / rarslave2.git / blobdiff