#!/usr/bin/env python
-# vim: set ts=4 sts=4 sw=4 textwidth=112 :
+# vim: set ts=4 sts=4 sw=4 textwidth=92:
+
+"""
+The main program of the rarslave project.
+
+This handles all of the commandline, configuration file, and option
+work. It gets the environment set up for a run using the RarslaveDetector
+class.
+"""
+
+__author__ = "Ira W. Snyder (devel@irasnyder.com)"
+__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)"
+__license__ = "GNU GPL v2 (or, at your option, any later version)"
+
+# rarslave.py -- a usenet autorepair and autoextract utility
+#
+# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
VERSION="2.0.0"
PROGRAM="rarslave2"
-import re, os, sys, optparse
-import par2parser
-import RarslaveConfig
-import RarslaveLogger
+import os, sys, optparse, logging
+import rsutil
+import RarslaveDetector
-# Global Variables
-(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4)
-(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4)
-config = RarslaveConfig.RarslaveConfig()
-logger = RarslaveLogger.RarslaveLogger ()
+# Global options from the rsutil.globals class
+options = rsutil.globals.options
+config = rsutil.globals.config
-# Global options to be set / used later.
-options = None
+# A tiny class to hold logging output until we're finished
+class DelayedLogger (object):
-class RarslaveExtractor (object):
+ """A small class to hold logging output until the program is finished running.
+ It emulates sys.stdout in the needed ways for the logging module."""
- def __init__ (self, type):
- self.type = type
- self.heads = []
+ def __init__ (self, output=sys.stdout.write):
+ self.__messages = []
+ self.__output = output
- def addHead (self, dir, head):
- assert os.path.isdir (dir)
- # REQUIRES that the dir is valid, but not that the file is valid, so that
- # we can move a file that doesn't exist yet.
- # FIXME: probably CAN add this back, since we should be running this AFTER repair.
- #assert os.path.isfile (os.path.join (dir, head))
+ def write (self, msg):
+ self.__messages.append (msg)
- full_head = os.path.join (dir, head)
- logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug)
- self.heads.append (full_head)
+ def flush (self):
+ pass
- def extract (self, todir=None):
- # Extract all heads of this set
+ def size (self):
+ """Returns the number of messages queued for printing"""
+ return len (self.__messages)
- # Create the directory $todir if it doesn't exist
- if todir != None and not os.path.isdir (todir):
- logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose)
- try:
- os.makedirs (todir)
- except OSError:
- logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal)
- return -EEXTRACT
+ def close (self):
+ """Print all messages, clear the queue"""
+ for m in self.__messages:
+ self.__output (m)
- # Extract all heads
- extraction_func = \
- { TYPE_OLDRAR : self.__extract_rar,
- TYPE_NEWRAR : self.__extract_rar,
- TYPE_ZIP : self.__extract_zip,
- TYPE_NOEXTRACT : self.__extract_noextract }[self.type]
+ self.__messages = []
- # Call the extraction function on each head
- for h in self.heads:
- if todir == None:
- # Run in the head's directory
- ret = extraction_func (h, os.path.dirname (h))
- else:
- ret = extraction_func (h, todir)
+# A tiny class used to find unique PAR2 sets
+class CompareSet (object):
- logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug)
+ """A small class used to find unique PAR2 sets"""
- # Check error code
- if ret != SUCCESS:
- logger.addMessage ('Failed extracting: %s' % h, RarslaveLogger.MessageType.Fatal)
- return -EEXTRACT
-
- return SUCCESS
-
- def __extract_rar (self, file, todir):
- assert os.path.isfile (file)
- assert os.path.isdir (todir)
-
- RAR_CMD = config.get_value ('commands', 'unrar')
-
- cmd = '%s \"%s\"' % (RAR_CMD, file)
- ret = run_command (cmd, todir)
-
- # Check error code
- if ret != 0:
- return -EEXTRACT
-
- return SUCCESS
-
- def __extract_zip (self, file, todir):
- ZIP_CMD = config.get_value ('commands', 'unzip')
-
- cmd = ZIP_CMD % (file, todir)
- ret = run_command (cmd)
-
- # Check error code
- if ret != 0:
- return -EEXTRACT
-
- return SUCCESS
-
- def __extract_noextract (self, file, todir):
- # Just move this file to the $todir, since no extraction is needed
- # FIXME: NOTE: mv will fail by itself if you're moving to the same dir!
- NOEXTRACT_CMD = config.get_value ('commands', 'noextract')
-
- cmd = NOEXTRACT_CMD % (file, todir)
- ret = run_command (cmd)
-
- # Check error code
- if ret != 0:
- return -EEXTRACT
-
- return SUCCESS
-
-
-
-class RarslaveRepairer (object):
- # Verify (and repair) the set
- # Make sure it worked, otherwise clean up and return failure
-
- def __init__ (self, dir, file, join=False):
- self.dir = dir # the directory containing the par2 file
- self.file = file # the par2 file
- self.join = join # True if the par2 set is 001 002 ...
-
- assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, file))
-
- def checkAndRepair (self):
- # Form the command:
- # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES]
- PAR2_CMD = config.get_value ('commands', 'par2repair')
-
- # Get set up
- basename = get_basename (self.file)
- all_files = find_likely_files (basename, self.dir)
- all_files.sort ()
- par2_files = find_par2_files (all_files)
-
- # assemble the command
- command = "%s \"%s\" " % (PAR2_CMD, self.file)
-
- for f in par2_files:
- if f != self.file:
- command += "\"%s\" " % os.path.split (f)[1]
-
- if self.join:
- for f in all_files:
- if f not in par2_files:
- command += "\"%s\" " % os.path.split (f)[1]
-
- # run the command
- ret = run_command (command, self.dir)
-
- # check the result
- if ret != 0:
- logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal)
- return -ECHECK
-
- return SUCCESS
-
-def run_command (cmd, indir=None):
- # Runs the specified command-line in the directory given (or, in the current directory
- # if none is given). It returns the status code given by the application.
-
- pwd = os.getcwd ()
-
- if indir != None:
- assert os.path.isdir (indir) # MUST be a directory!
- os.chdir (indir)
-
- # FIXME: re-enable this after testing
- print 'RUNNING (%s): %s' % (indir, cmd)
- return SUCCESS
-
- # return os.system (cmd)
-
-
-def full_abspath (p):
- return os.path.abspath (os.path.expanduser (p))
-
-def get_basename (name):
- """Strips most kinds of endings from a filename"""
-
- regex = config.get_value ('regular expressions', 'basename_regex')
- r = re.compile (regex, re.IGNORECASE)
- done = False
-
- while not done:
- done = True
-
- if r.match (name):
- g = r.match (name).groups()
- name = g[0]
- done = False
-
- return name
+ def __init__ (self, dir, p2file):
+ self.dir = dir
+ self.p2file = p2file
-def find_likely_files (name, dir):
- """Finds files which are likely to be part of the set corresponding
- to $name in the directory $dir"""
+ self.basename = rsutil.common.get_basename (self.p2file)
+ self.name_matches = rsutil.common.find_name_matches (self.dir, self.basename)
- if not os.path.isdir (os.path.abspath (dir)):
- raise ValueError # bad directory given
-
- dir = os.path.abspath (dir)
- ename = re.escape (name)
- regex = re.compile ('^%s.*$' % (ename, ))
+ def __eq__ (self, rhs):
+ return (self.dir == rhs.dir) \
+ and (self.basename == rhs.basename) \
+ and rsutil.common.list_eq (self.name_matches, rhs.name_matches)
- return [f for f in os.listdir (dir) if regex.match (f)]
-def find_par2_files (files):
- """Find all par2 files in the list $files"""
+def find_all_par2_files (dir):
+ """Finds all par2 files in the given directory.
- PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex')
- regex = re.compile (PAR2_REGEX, re.IGNORECASE)
- return [f for f in files if regex.match (f)]
+ dir -- the directory in which to search for PAR2 files
-def find_all_par2_files (dir):
- """Finds all par2 files in a directory"""
- # NOTE: does NOT return absolute paths
+ NOTE: does not return absolute paths"""
if not os.path.isdir (os.path.abspath (dir)):
raise ValueError # bad directory given
dir = os.path.abspath (dir)
files = os.listdir (dir)
- return find_par2_files (files)
-
-def has_extension (f, ext):
- """Checks if f has the extension ext"""
-
- if ext[0] != '.':
- ext = '.' + ext
-
- ext = re.escape (ext)
- regex = re.compile ('^.*%s$' % (ext, ), re.IGNORECASE)
- return regex.match (f)
-
-def find_extraction_heads (dir, files):
- """Takes a list of possible files and finds likely heads of
- extraction."""
-
- # NOTE: perhaps this should happen AFTER repair is
- # NOTE: successful. That way all files would already exist
-
- # According to various sources online:
- # 1) pre rar-3.0: .rar .r00 .r01 ...
- # 2) post rar-3.0: .part01.rar .part02.rar
- # 3) zip all ver: .zip
-
- extractor = None
- p2files = find_par2_files (files)
-
- # Old RAR type, find all files ending in .rar
- if is_oldrar (files):
- extractor = RarslaveExtractor (TYPE_OLDRAR)
- regex = re.compile ('^.*\.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
-
- if is_newrar (files):
- extractor = RarslaveExtractor (TYPE_NEWRAR)
- regex = re.compile ('^.*\.part01.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
-
- if is_zip (files):
- extractor = RarslaveExtractor (TYPE_ZIP)
- regex = re.compile ('^.*\.zip$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
-
- if is_noextract (files):
- # Use the Par2 Parser (from cfv) here to find out what files are protected.
- # Since these are not being extracted, they will be mv'd to another directory
- # later.
- extractor = RarslaveExtractor (TYPE_NOEXTRACT)
-
- for f in p2files:
- done = False
- try:
- prot_files = par2parser.get_protected_files (dir, f)
- done = True
- except: #FIXME: add the actual exceptions
- logger.addMessage ('Error parsing PAR2 file: %s', f)
- continue
-
- if done:
- break
-
- if done:
- for f in prot_files:
- extractor.addHead (dir, f)
- else:
- logger.addMessage ('Error parsing all PAR2 files in this set ...')
-
- # Make sure we found the type
- if extractor == None:
- logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0],
- RarslaveLogger.MessageType.Fatal)
-
- # No-heads here, but it's better than failing completely
- extractor = RarslaveExtractor (TYPE_NOEXTRACT)
-
- return extractor
-
-def is_oldrar (files):
- for f in files:
- if has_extension (f, '.r00'):
- return True
-
- return False
-
-def is_newrar (files):
- for f in files:
- if has_extension (f, '.part01.rar'):
- return True
-
- return False
-
-def is_zip (files):
- for f in files:
- if has_extension (f, '.zip'):
- return True
-
- return False
-
-def is_noextract (files):
- # Type that needs no extraction.
- # TODO: Add others ???
- for f in files:
- if has_extension (f, '.001'):
- return True
-
- return False
-
-def find_deleteable_files (files):
- # Deleteable types regex should come from the config
- dfiles = []
- DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex')
- dregex = re.compile (DELETE_REGEX, re.IGNORECASE)
-
- return [f for f in files if dregex.match (f)]
-
-def printlist (li):
- for f in li:
- print f
-
-class PAR2Set (object):
-
- dir = None
- file = None
- likely_files = []
-
- def __init__ (self, dir, file):
- assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, file))
-
- self.dir = dir
- self.file = file
-
- basename = get_basename (file)
- self.likely_files = find_likely_files (basename, dir)
-
- def __list_eq (self, l1, l2):
-
- if len(l1) != len(l2):
- return False
-
- for e in l1:
- if e not in l2:
- return False
-
- return True
-
- def __eq__ (self, rhs):
- return self.__list_eq (self.likely_files, rhs.likely_files)
-
- def run_all (self):
- par2files = find_par2_files (self.likely_files)
- par2head = par2files[0]
-
- join = is_noextract (self.likely_files)
-
- # Repair Stage
- repairer = RarslaveRepairer (self.dir, par2head, join)
- ret = repairer.checkAndRepair ()
-
- if ret != SUCCESS:
- logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
- return -ECHECK
-
- # Extraction Stage
- EXTRACT_DIR = options.extract_dir
- extractor = find_extraction_heads (self.dir, self.likely_files)
- ret = extractor.extract (EXTRACT_DIR)
-
- if ret != SUCCESS:
- logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
- return -EEXTRACT
-
- # Deletion Stage
- DELETE_INTERACTIVE = options.interactive
- deleteable_files = find_deleteable_files (self.likely_files)
- ret = delete_list (deleteable_files, DELETE_INTERACTIVE)
-
- if ret != SUCCESS:
- logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
- return -EDELETE
-
- logger.addMessage ('Successfully completed: %s' % par2head)
- return SUCCESS
-
-def delete_list (files, interactive=False):
- # Delete a list of files
-
- done = False
- valid_y = ['Y', 'YES']
- valid_n = ['N', 'NO']
-
- if interactive:
- while not done:
- print 'Do you want to delete the following?:'
- s = raw_input ('Delete [y/N]: ').upper()
-
- if s in valid_y + valid_n:
- done = True
-
- if s in valid_n:
- return SUCCESS
-
- for f in files:
- # FIXME: re-enable this in production
- # os.remove (f)
- print 'rm \"%s\"' % f
-
- return SUCCESS
-
+ return rsutil.common.find_par2_files (files)
def generate_all_parsets (dir):
- # Generate all parsets in the given directory.
+ """Generate all parsets in the given directory
+
+ dir -- the directory in which to search"""
assert os.path.isdir (dir) # Directory MUST be valid
p2files = find_all_par2_files (dir)
for f in p2files:
- p = PAR2Set (dir, f)
+ p = CompareSet (dir, f)
if p not in parsets:
parsets.append (p)
- return parsets
+ return [(p.dir, p.p2file) for p in parsets]
def check_required_progs():
"""Check if the required programs are installed"""
shell_not_found = 32512
needed = []
- if run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found:
+ if rsutil.common.run_command ('par2repair --help > /dev/null 2>&1') == shell_not_found:
needed.append ('par2repair')
- if run_command ('unrar --help > /dev/null 2>&1') == shell_not_found:
+ if rsutil.common.run_command ('unrar --help > /dev/null 2>&1') == shell_not_found:
needed.append ('unrar')
- if run_command ('unzip --help > /dev/null 2>&1') == shell_not_found:
+ if rsutil.common.run_command ('unzip --help > /dev/null 2>&1') == shell_not_found:
needed.append ('unzip')
if needed:
sys.exit(1)
def run_options (options):
- options.work_dir = full_abspath (options.work_dir)
+ """Process all of the commandline options, doing thing such as printing the
+ version number, etc."""
+
+ # Fix directories
+ options.work_dir = rsutil.common.full_abspath (options.work_dir)
+
+ # Make sure that the directory is valid
+ if not os.path.isdir (options.work_dir):
+ sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir)
+ sys.stderr.write ('option to override the working directory temporarily, or edit the\n')
+ sys.stderr.write ('configuration file to override the working directory permanently.\n')
+ sys.exit (1)
if options.extract_dir != None:
- options.extract_dir = full_abspath (options.extract_dir)
+ options.extract_dir = rsutil.common.full_abspath (options.extract_dir)
if options.version:
print PROGRAM + ' - ' + VERSION
if options.write_def_config:
config.write_config (default=True)
+ sys.exit (0)
if options.write_config:
config.write_config ()
+ sys.exit (0)
def find_loglevel (options):
+ """Find the log level that should be printed by the logging class"""
loglevel = options.verbose - options.quiet
- if loglevel < RarslaveLogger.MessageType.Fatal:
- loglevel = RarslaveLogger.MessageType.Fatal
-
- if loglevel > RarslaveLogger.MessageType.Debug:
- loglevel = RarslaveLogger.MessageType.Debug
-
- return loglevel
-
-def printMessageTable (loglevel):
-
- if logger.hasFatalMessages ():
- print '\nFatal Messages\n' + '=' * 80
- logger.printLoglevel (RarslaveLogger.MessageType.Fatal)
+ if loglevel > 1:
+ loglevel = 1
- if loglevel == RarslaveLogger.MessageType.Fatal:
- return
+ if loglevel < -3:
+ loglevel = -3
- if logger.hasNormalMessages ():
- print '\nNormal Messages\n' + '=' * 80
- logger.printLoglevel (RarslaveLogger.MessageType.Normal)
+ LEVELS = { 1 : logging.DEBUG,
+ 0 : logging.INFO,
+ -1: logging.WARNING,
+ -2: logging.ERROR,
+ -3: logging.CRITICAL
+ }
- if loglevel == RarslaveLogger.MessageType.Normal:
- return
-
- if logger.hasVerboseMessages ():
- print '\nVerbose Messages\n' + '=' * 80
- logger.printLoglevel (RarslaveLogger.MessageType.Verbose)
-
- if loglevel == RarslaveLogger.MessageType.Verbose:
- return
-
- if logger.hasDebugMessages ():
- print '\nDebug Messages\n' + '=' * 80
- logger.printLoglevel (RarslaveLogger.MessageType.Debug)
-
- return
+ return LEVELS [loglevel]
def main ():
+ # Setup the logger
+ logger = DelayedLogger ()
+ logging.basicConfig (stream=logger, level=logging.WARNING, \
+ format='%(levelname)-8s %(message)s')
+
# Build the OptionParser
parser = optparse.OptionParser()
- parser.add_option('-n', '--not-recursive',
- action='store_false', dest='recursive',
- default=config.get_value('options', 'recursive'),
+ parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive',
+ default=rsutil.common.config_get_value('options', 'recursive'),
help="Don't run recursively")
- parser.add_option('-d', '--work-dir',
- dest='work_dir', type='string',
- default=config.get_value('directories', 'working_directory'),
+ parser.add_option('-d', '--work-dir', dest='work_dir', type='string',
+ default=rsutil.common.config_get_value('directories', 'working_directory'),
help="Start running at DIR", metavar='DIR')
- parser.add_option('-e', '--extract-dir',
- dest='extract_dir', type='string',
- default=config.get_value('directories', 'extract_directory'),
+ parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string',
+ default=rsutil.common.config_get_value('directories', 'extract_directory'),
help="Extract to DIR", metavar='DIR')
parser.add_option('-p', '--check-required-programs',
default=False, help="Write out the current config")
parser.add_option('-i', '--interactive', dest='interactive', action='store_true',
- default=config.get_value('options', 'interactive'),
+ default=rsutil.common.config_get_value('options', 'interactive'),
help="Confirm before removing files")
parser.add_option('-q', '--quiet', dest='quiet', action='count',
# Parse the given options
global options
- (options, args) = parser.parse_args()
+ (rsutil.globals.options, args) = parser.parse_args()
+ options = rsutil.globals.options
# Run any special actions that are needed on these options
run_options (options)
- # Find the loglevel using the options given
- loglevel = find_loglevel (options)
+ # Find the loglevel using the options given
+ logging.getLogger().setLevel (find_loglevel (options))
# Run recursively
if options.recursive:
for (dir, subdirs, files) in os.walk (options.work_dir):
parsets = generate_all_parsets (dir)
- for p in parsets:
- p.run_all ()
+ for (p2dir, p2file) in parsets:
+ detector = RarslaveDetector.RarslaveDetector (p2dir, p2file)
+ ret = detector.runMatchingTypes ()
# Non-recursive
else:
parsets = generate_all_parsets (options.work_dir)
- for p in parsets:
- p.run_all ()
+ for (p2dir, p2file) in parsets:
+ detector = RarslaveDetector.RarslaveDetector (p2dir, p2file)
+ ret = detector.runMatchingTypes ()
# Print the results
- printMessageTable (loglevel)
+ if logger.size () > 0:
+ print '\nLog\n' + '=' * 80
+ logger.close ()
# Done!
return 0