From feeefeb8ea2f1e4724424d43c0eb872aee4743c2 Mon Sep 17 00:00:00 2001 From: "Ira W. Snyder" Date: Sat, 1 Nov 2008 23:02:39 -0700 Subject: [PATCH] Major Update This changes the structure of the entire program, making it much simpler to maintain. The old structure was needlessly complicated by improper use of packages. This version is lacking a few features that were present in the original version, but nothing that I ever used. Signed-off-by: Ira W. Snyder --- PAR2Set/Base.py | 289 +++++++++++---------- PAR2Set/CompareSet.py | 83 ++++++ PAR2Set/ExtractFirstBase.py | 57 +++-- PAR2Set/ExtractFirstNewRAR.py | 24 +- PAR2Set/ExtractFirstOldRAR.py | 24 +- PAR2Set/Join.py | 70 ++--- PAR2Set/NewRAR.py | 38 +-- PAR2Set/NoExtract.py | 80 ++---- PAR2Set/OldRAR.py | 38 +-- PAR2Set/ZIP.py | 38 ++- PAR2Set/__init__.py | 33 ++- PAR2Set/par2parser.py | 143 +++++++++++ PAR2Set/utils.py | 118 +++++++++ RarslaveDetector.py | 126 --------- rarslave-test.py | 103 -------- rarslave.py | 465 +++++++++++++++++++--------------- rsutil/__init__.py | 7 - rsutil/common.py | 164 ------------ rsutil/config.py | 171 ------------- rsutil/globals.py | 44 ---- rsutil/par2parser.py | 139 ---------- setup.py | 5 +- 22 files changed, 912 insertions(+), 1347 deletions(-) create mode 100644 PAR2Set/CompareSet.py create mode 100644 PAR2Set/par2parser.py create mode 100644 PAR2Set/utils.py delete mode 100644 RarslaveDetector.py delete mode 100644 rarslave-test.py delete mode 100644 rsutil/__init__.py delete mode 100644 rsutil/common.py delete mode 100644 rsutil/config.py delete mode 100644 rsutil/globals.py delete mode 100644 rsutil/par2parser.py diff --git a/PAR2Set/Base.py b/PAR2Set/Base.py index a5d3dcb..599827e 100644 --- a/PAR2Set/Base.py +++ b/PAR2Set/Base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: +# vim: set ts=4 sts=4 sw=4 textwidth=80: """ Holds the PAR2Set base class @@ -27,199 +27,196 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import re -import os -import logging +import re, os, logging +from subprocess import CalledProcessError +from PAR2Set import CompareSet, utils -import rsutil.common - -# This is a fairly generic class which does all of the major things that a PAR2 -# set will need to have done to be verified and extracted. For most "normal" types -# you won't need to override hardly anything. -# -# It is ok to override other functions if the need arises, just make sure that you -# understand why things are done the way that they are in the original versions. +# This is a generic class which handles all of the major operations that a PAR2 +# set will need to be repaired, extracted, and cleaned up. For most subclasses, +# you shouldn't need to do very much. # -# Assumptions made in the runVerifyAndRepair(), runExtract() and runDelete() functions: -# ============================================================================== -# The state of self.name_matched_files, self.prot_matched_files, and self.all_files -# will be consistent with the real, in-filesystem state at the time that they are -# called. This is the reason that runAll() calls update_matches() after running each -# operation that will possibly change the filesystem. -# -# Required overrides: -# ============================================================================== -# find_extraction_heads () -# extraction_function () +# When the repair(), extract(), and delete() methods run, all of the class +# instance variables MUST match the in-filesystem state # +# To create a new subclass: +# 1) Override the detect() method to detect your set type, raise TypeError if +# this is not your type +# 2) Optionally, override extract() to extract your set +# 3) Override any other methods where your set differs from the Base +# implementation + +class Base(object): + + # Constructor + # @cs an instance of CompareSet + def __init__(self, cs, options): + + # Save the options + self.options = options + + # The directory and parity file + self.directory = cs.directory + self.parityFile = cs.parityFile -class Base (object): - """Base class for all PAR2Set types""" + # The base name of the parity file (for matching) + self.baseName = cs.baseName - # Instance Variables - # ========================================================================== - # dir -- The directory this set lives in - # p2file -- The starting PAR2 file - # basename -- The basename of the set, guessed from the PAR2 file - # all_p2files -- All PAR2 files of the set, guessed from the PAR2 file name only - # name_matched_files -- Files in this set, guessed by name only - # prot_matched_files -- Files in this set, guessed by parsing the PAR2 only + # Files that match by name only + self.similarlyNamedFiles = cs.similarlyNamedFiles - def __init__ (self, dir, p2file): - """Default constructor for all PAR2Set types + # PAR2 files from the files matched by name + self.PAR2Files = utils.findMatches(r'^.*\.par2', self.similarlyNamedFiles) - dir -- a directory - p2file -- a PAR2 file inside the given directory""" + # All of the files protected by this set + self.protectedFiles = cs.protectedFiles - assert os.path.isdir (dir) - assert os.path.isfile (os.path.join (dir, p2file)) + # Create a set of all the combined files + self.allFiles = set(self.similarlyNamedFiles + self.protectedFiles) - # The real "meat" of the class - self.dir = dir - self.p2file = p2file - self.basename = rsutil.common.get_basename (p2file) + # Run the detection + # NOTE: Python calls "down" into derived classes automatically + # WARNING: This will raise TypeError if it doesn't match + self.detect() - # Find files that match by name only - self.name_matched_files = rsutil.common.find_name_matches (self.dir, self.basename) + ############################################################################ - # Find all par2 files for this set using name matches - self.all_p2files = rsutil.common.find_par2_files (self.name_matched_files) + def __str__(self): + return '%s: %s' % (self.__class__.__name__, self.parityFile) - # Try to get the protected files for this set - self.prot_matched_files = rsutil.common.parse_all_par2 (self.dir, self.p2file, self.all_p2files) + ############################################################################ - # Setup the all_files combined set (for convenience only) - self.all_files = rsutil.common.no_duplicates (self.name_matched_files + self.prot_matched_files) + def __eq__(self, rhs): + return self.allFiles == rhs.allFiles - def __eq__ (self, rhs): - """Check for equality between PAR2Set types""" + ############################################################################ - return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \ - rsutil.common.list_eq (self.name_matched_files, rhs.name_matched_files) and \ - rsutil.common.list_eq (self.prot_matched_files, rhs.prot_matched_files) + # Run all operations + def run(self): - def update_matches (self): - """Updates the contents of instance variables with the current in-filesystem state. - This should be run after any operation which can create or delete files.""" + # Repair Stage + try: + self.repair() + except (CalledProcessError, OSError): + logging.critical('Repair stage failed for: %s' % self) + raise - self.name_matched_files = rsutil.common.find_name_matches (self.dir, self.basename) - self.all_files = rsutil.common.no_duplicates (self.name_matched_files + self.prot_matched_files) + self.updateFilesystemState() - def runVerifyAndRepair (self): - """Verify and Repair a PAR2Set. This is done using the par2repair command by - default""" + # Extraction Stage + try: + self.extract () + except (CalledProcessError, OSError): + logging.critical('Extraction stage failed for: %s' % self) + raise - rsutil.common.run_command(['par2repair'] + self.all_p2files, self.dir) + self.updateFilesystemState() - def find_deleteable_files (self): - """Find all files which are deletable by using the regular expression from the - configuration file""" + # Deletion Stage + try: + self.delete () + except (CalledProcessError, OSError): + logging.critical('Deletion stage failed for: %s' % self) + raise - DELETE_REGEX = rsutil.common.config_get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + logging.debug ('Successfully completed: %s' % self) - return [f for f in self.all_files if dregex.match (f)] + ############################################################################ - def delete_list_of_files (self, dir, files, interactive=False): - """Attempt to delete all files given + # Run the repair + def repair(self): - dir -- the directory where the files live - files -- the filenames themselves - interactive -- prompt before deleteion""" + # This is overly simple, but it works great for almost everything + utils.runCommand(['par2repair'] + self.PAR2Files, self.directory) - assert os.path.isdir (dir) + ############################################################################ - # If we are interactive, decide yes or no - if interactive: - while True: - print 'Do you want to delete the following?:' - for f in files: - print f + # Run the extraction + def extract(self): + pass - s = raw_input('Delete [y/N]: ').upper() + ############################################################################ - # If we got a valid no answer, leave now - if s in ['N', 'NO', '']: - return + def findDeletableFiles(self): - # If we got a valid yes answer, delete them - if s in ['Y', 'YES']: - break + regex = r'^.*\.(par2|\d|\d\d\d|rar|r\d\d|zip)$' + files = utils.findMatches(regex, self.allFiles) - # Not a good answer, ask again - print 'Invalid response' + return files - # We got a yes answer (or are non-interactive), delete the files - for f in files: + ############################################################################ - # Get the full filename - fullname = os.path.join(dir, f) + # Run the deletion + def delete(self): - # Delete the file - try: - os.remove(fullname) - logging.debug('Deleting: %s' % fullname) - except OSError: - logging.error('Failed to delete: %s' % fullname) + # If we aren't to run delete, don't + if self.options.delete == False: + return - def runDelete (self): - """Run the delete operation and return the result""" + # Find all deletable files + files = self.findDeletableFiles() + files.sort() - deleteable_files = self.find_deleteable_files () - ret = self.delete_list_of_files (self.dir, deleteable_files, \ - rsutil.common.options_get_value ('interactive')) + # If we are interactive, get the user's response, 'yes' or 'no' + if self.options.interactive: + while True: + print 'Do you want to delete the following?:' + for f in files: + print f - return ret + s = raw_input('Delete [y/N]: ').upper() - def runAll (self): - """Run all of the major sections in the class: repair, extraction, and deletion.""" + # If we got a valid no answer, leave now + if s in ['N', 'NO', '']: + return - # Repair Stage - try: - self.runVerifyAndRepair () - except (RuntimeError, OSError): - logging.critical('Repair stage failed for: %s' % self.p2file) - raise + # If we got a valid yes answer, delete them + if s in ['Y', 'YES']: + break - self.update_matches() + # Not a good answer, ask again + print 'Invalid response' - # Extraction Stage - try: - self.runExtract () - except (RuntimeError, OSError): - logging.critical('Extraction stage failed for: %s' % self.p2file) - raise + # We got a yes answer (or are non-interactive), delete the files + for f in files: - self.update_matches () + # Get the full filename + fullname = os.path.join(self.directory, f) - # Deletion Stage - try: - self.runDelete () - except (RuntimeError, OSError): - logging.critical('Deletion stage failed for: %s' % self.p2file) - raise + # Delete the file + try: + os.remove(fullname) + print 'rm', fullname + logging.debug('Deleting: %s' % fullname) + except OSError: + logging.error('Failed to delete: %s' % fullname) - logging.info ('Successfully completed: %s' % self.p2file) + ############################################################################ - def runExtract (self): - """Extract all heads of this set and return the result""" + # Detect if the given files make up a set of this type + # + # Raise a TypeError if there is no match. This is meant to be called + # from the constructor, from which we cannot return a value... + def detect(self): - # Call the extraction function on each head - for h in self.find_extraction_heads (): - full_head = rsutil.common.full_abspath (os.path.join (self.dir, h)) - self.extraction_function (full_head, self.dir) + # This must be overridden by the subclasses that actually implement + # the functionality of rarslave - def find_extraction_heads (self): - """Find all extraction heads associated with this set. This must be - overridden for the associated PAR2Set derived class to work.""" + # The original implementation used ONLY the following here: + # self.similarlyNamedFiles + # self.protectedFiles + raise TypeError - assert False # You MUST override this on a per-type basis + ############################################################################ - def extraction_function (self, file, todir): - """Extract a single head of this PAR2Set's type. + # Update the class' state to match the current filesystem state + # + # This must be called after any operation which can create or delete files + # which are relevant to repair(), extract(), and delete() + def updateFilesystemState(self): - file -- the full path to the file to be extracted - todir -- the directory to extract to""" + self.similarlyNamedFiles = utils.findFileNameMatches(self.directory, + self.baseName) + self.allFiles = set(self.similarlyNamedFiles + self.protectedFiles) - assert False # You MUST override this on a per-type basis + ############################################################################ diff --git a/PAR2Set/CompareSet.py b/PAR2Set/CompareSet.py new file mode 100644 index 0000000..5f99f66 --- /dev/null +++ b/PAR2Set/CompareSet.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=80: + +""" +Holds the PAR2Set CompareSet class + +This class is used in the detection of PAR2Sets, and is provided here +to reduce the coupling between rarslave.py and the PAR2Set classes. + +You should create one of these, and then send it to any of the other PAR2Set +classes' constructor. +""" + +__author__ = "Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2008 Ira W. Snyder (devel@irasnyder.com)" +__license__ = "GNU GPL v2 (or, at your option, any later version)" + +# CompareSet.py +# +# Copyright (C) 2008 Ira W. Snyder (devel@irasnyder.com) +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from PAR2Set import utils, par2parser + +class CompareSet(object): + + ############################################################################ + + def __init__(self, directory, parityFile): + self.directory = directory + self.parityFile = parityFile + + self.baseName = utils.getBasename(parityFile) + self.similarlyNamedFiles = utils.findFileNameMatches(directory, self.baseName) + + # WARNING: This is likely to raise an exception! + # WARNING: Don't forget to check for it :) + self.protectedFiles = par2parser.getProtectedFiles(directory, parityFile) + + # Sort so the compare works as expected + self.similarlyNamedFiles.sort() + self.protectedFiles.sort() + + ############################################################################ + + def __eq__(self, rhs): + + return self.directory == rhs.directory \ + and self.baseName == rhs.baseName \ + and self.similarlyNamedFiles == rhs.similarlyNamedFiles \ + and self.protectedFiles == rhs.protectedFiles + + ############################################################################ + + def __str__(self): + s = '%s\n' % repr(self) + s += 'directory: %s\n' % self.directory + s += 'parityFile: %s\n' % self.parityFile + s += 'baseName: %s\n' % self.baseName + s += 'similarlyNamedFiles:\n' + for f in self.similarlyNamedFiles: + s += '-> %s\n' % f + s += 'protectedFiles:\n' + for f in self.protectedFiles: + s += '-> %s\n' % f + + return s + + ############################################################################ + diff --git a/PAR2Set/ExtractFirstBase.py b/PAR2Set/ExtractFirstBase.py index 7acfc63..c62d842 100644 --- a/PAR2Set/ExtractFirstBase.py +++ b/PAR2Set/ExtractFirstBase.py @@ -28,40 +28,41 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import logging -import PAR2Set.Base -import rsutil.common +from subprocess import CalledProcessError +from PAR2Set import Base -class ExtractFirstBase (PAR2Set.Base.Base): - """Base class for PAR2Set types which run the extraction routine before - the repair routine""" +class ExtractFirstBase(Base): - def runAll (self): - """Run the extraction, repair, and delete stages""" + ############################################################################ - # Extraction Stage - try: - self.runExtract() - except (RuntimeError, OSError): - logging.critical('Extraction stage failed for: %s' % self.p2file) - raise + def run(self): - self.update_matches () + # Extraction Stage + try: + self.extract () + except (CalledProcessError, OSError): + logging.critical('Extraction stage failed for: %s' % self) + raise - # Repair Stage - try: - self.runVerifyAndRepair() - except (RuntimeError, OSError): - logging.critical('Repair stage failed for: %s' % self.p2file) - raise + self.updateFilesystemState() - self.update_matches () + # Repair Stage + try: + self.repair() + except (CalledProcessError, OSError): + logging.critical('Repair stage failed for: %s' % self) + raise - # Deletion Stage - try: - self.runDelete() - except (RuntimeError, OSError): - logging.critical('Deletion stage failed for: %s' % self.p2file) - raise + self.updateFilesystemState() - logging.info ('Successfully completed: %s' % self.p2file) + # Deletion Stage + try: + self.delete () + except (CalledProcessError, OSError): + logging.critical('Deletion stage failed for: %s' % self) + raise + + logging.info ('Successfully completed: %s' % self) + + ############################################################################ diff --git a/PAR2Set/ExtractFirstNewRAR.py b/PAR2Set/ExtractFirstNewRAR.py index 79fb727..723d1ed 100644 --- a/PAR2Set/ExtractFirstNewRAR.py +++ b/PAR2Set/ExtractFirstNewRAR.py @@ -42,23 +42,17 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import PAR2Set.ExtractFirstBase -import PAR2Set.NewRAR -import rsutil.common +from PAR2Set import ExtractFirstBase, NewRAR, utils +class ExtractFirstNewRAR(ExtractFirstBase, NewRAR): -def detector (name_files, prot_files): - """Detects a ExtractFirstNewRAR set""" + def detect(self): + regex = r'^.*\.part0*1\.rar$' + m1 = utils.hasAMatch(regex, self.similarlyNamedFiles) + m2 = utils.hasAMatch(regex, self.protectedFiles) - return rsutil.common.has_a_match ('^.*\.part0*1\.rar$', name_files) \ - and not rsutil.common.has_a_match ('^.*\.part0*1\.rar$', prot_files) + if m1 and not m2: + return - -class ExtractFirstNewRAR (PAR2Set.ExtractFirstBase.ExtractFirstBase, \ - PAR2Set.NewRAR.NewRAR): - - """Class for new-style rar sets which must be extracted before repair""" - - def __repr__ (self): - return 'EXTRACTFIRST NEWRAR' + raise TypeError diff --git a/PAR2Set/ExtractFirstOldRAR.py b/PAR2Set/ExtractFirstOldRAR.py index 26ed997..500a46c 100644 --- a/PAR2Set/ExtractFirstOldRAR.py +++ b/PAR2Set/ExtractFirstOldRAR.py @@ -43,23 +43,17 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import PAR2Set.ExtractFirstBase -import PAR2Set.OldRAR -import rsutil.common +from PAR2Set import ExtractFirstBase, OldRAR, utils +class ExtractFirstOldRAR(ExtractFirstBase, OldRAR): -def detector (name_files, prot_files): - """Detects a ExtractFirstOldRAR set""" + def detect(self): + regex = r'^.*\.r00$' + m1 = utils.hasAMatch(regex, self.similarlyNamedFiles) + m2 = utils.hasAMatch(regex, self.protectedFiles) - return rsutil.common.has_a_match ('^.*\.r00$', name_files) \ - and not rsutil.common.has_a_match ('^.*\.r00$', prot_files) + if m1 and not m2: + return - -class ExtractFirstOldRAR (PAR2Set.ExtractFirstBase.ExtractFirstBase, \ - PAR2Set.OldRAR.OldRAR): - - """Class for old-style rar sets which must be extracted before repair""" - - def __repr__ (self): - return 'EXTRACTFIRST OLDRAR' + raise TypeError diff --git a/PAR2Set/Join.py b/PAR2Set/Join.py index 1dd680a..0970d87 100644 --- a/PAR2Set/Join.py +++ b/PAR2Set/Join.py @@ -42,65 +42,41 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import re -import os -import logging -import PAR2Set.Base -import rsutil.common +import os, re, logging +from PAR2Set import Base, utils +class Join(Base): -def detector (name_files, prot_files): - """Detects a Join set""" + ############################################################################ - return rsutil.common.has_a_match ('^.*\.\d\d\d$', name_files) \ - and not rsutil.common.has_a_match ('^.*\.\d\d\d$', prot_files) + def detect(self): + regex = r'^.*\.\d\d\d$' + m1 = utils.hasAMatch(regex, self.similarlyNamedFiles) + m2 = utils.hasAMatch(regex, self.protectedFiles) -class Join (PAR2Set.Base.Base): + # This is a good match if this criteria is met + if m1 and not m2: + return - """Class for normal joined-file sets""" + raise TypeError - def __repr__ (self): - return 'JOIN' + ############################################################################ - def find_joinfiles (self): - """Finds files which contain data to be joined together""" + def repair(self): - return rsutil.common.find_matches ('^.*\.\d\d\d$', self.name_matched_files) + regex = r'^.*\.\d\d\d$' + files = utils.findMatches(regex, self.similarlyNamedFiles) + utils.runCommand(['par2repair'] + self.PAR2Files + files, self.directory) - def runVerifyAndRepair (self): - """Verify and Repair a PAR2Set. This version extends the PAR2Set.Base.Base - version by adding the datafiles to be joined at the end of the command - line. + ############################################################################ - This is done using the par2repair command by default""" + def findDeletableFiles(self): - rsutil.common.run_command(['par2repair'] + self.all_p2files + self.find_joinfiles(), self.dir) + files = Base.findDeletableFiles(self) + files = [f for f in files if f not in self.protectedFiles] - def find_deleteable_files (self): - """Find all files which are deletable by using the regular expression from the - configuration file""" + return files - DELETE_REGEX = rsutil.common.config_get_value ('regular expressions', 'delete_regex') - dregex = re.compile (DELETE_REGEX, re.IGNORECASE) + ############################################################################ - return [f for f in self.all_files if dregex.match (f) and \ - f not in self.prot_matched_files] - - def find_extraction_heads (self): - """Find the extraction heads. Since this should not be an extractable set, - we return the files which are protected directly by the PAR2 files.""" - - return self.prot_matched_files - - def extraction_function (self, file, todir): - """Extract a single file of the Join type. - - file -- the file to extract - todir -- the directory to extract to - - This command ignores the extraction if file and todir+file are the same - file. This keeps things like mv working smoothly.""" - - # The Join type doesn't need any extraction - pass diff --git a/PAR2Set/NewRAR.py b/PAR2Set/NewRAR.py index 7c8f25b..fc55d1c 100644 --- a/PAR2Set/NewRAR.py +++ b/PAR2Set/NewRAR.py @@ -40,37 +40,23 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import os -import PAR2Set.Base -import rsutil.common +from PAR2Set import Base, utils +class NewRAR(Base): -def detector (name_files, prot_files): - """Detector for the NewRAR type""" + ############################################################################ - return rsutil.common.has_a_match ('^.*\.part0*1\.rar$', prot_files) + def detect(self): + if not utils.hasAMatch('^.*\.part0*1\.rar$', self.protectedFiles): + raise TypeError + ############################################################################ -class NewRAR (PAR2Set.Base.Base): + def extract(self): + files = utils.findMatches('^.*\.part0*1\.rar$', self.protectedFiles) - """Class for new-style rar sets""" + for f in files: + utils.runCommand(['unrar', 'x', '-o+', f], self.directory) - def __repr__ (self): - return 'NEWRAR' - - def find_extraction_heads (self): - """Find the files to start extraction from. These end in '.part0*1.rar'""" - - return rsutil.common.find_matches ('^.*\.part0*1\.rar$', self.all_files) - - def extraction_function (self, file, todir): - """Extract a single rar file to the directory todir. - - file -- the file to be extracted - todir -- the directory to extract into""" - - assert os.path.isfile (file) - assert os.path.isdir (todir) - - rsutil.common.run_command(['unrar', 'x', '-o+', file], todir) + ############################################################################ diff --git a/PAR2Set/NoExtract.py b/PAR2Set/NoExtract.py index 9b4ea46..fcc01f8 100644 --- a/PAR2Set/NoExtract.py +++ b/PAR2Set/NoExtract.py @@ -44,70 +44,44 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import logging -import PAR2Set.Base -import rsutil.common +from PAR2Set import Base, utils +class NoExtract(Base): -def detector (name_files, prot_files): - """Detector for the NoExtract type""" + ############################################################################ - EXTRACT_REGEX = rsutil.common.config_get_value ('regular expressions', 'extractable_regex') - return not rsutil.common.has_a_match (EXTRACT_REGEX, prot_files) + def detect(self): + regex = r'^.+\.(rar|r\d\d|\d\d\d|zip)$' -class NoExtract (PAR2Set.Base.Base): + # This set has no extractable files + if utils.hasAMatch(regex, self.protectedFiles): + raise TypeError - """Class for sets that do not need to be extracted. Note that this is not a - base class, this is a working class.""" + ############################################################################ - def __repr__ (self): - return 'NoExtract' + # A NoExtract set is very different from the other types of sets in terms + # of filesystem state. It is very likely that normal name matching will not + # work correctly. + # + # We override this to try to match on every name that is protected by this set, + # which will detect the .1 files that are produced when repairing + def updateFilesystemState(self): - def update_matches (self): - """Updates the contents of instance variables with the current in-filesystem state. - This should be run after any operation which can create or delete files.""" + self.similarlyNamedFiles = utils.findFileNameMatches(self.directory, self.baseName) - # A NoExtract set is very different from the other sets in this regard. Since we - # don't really have a single file that is protected, it's not likely that "normal" - # name matching will work as expected. - # - # Because of this, we will try to find name matches for every single name that is - # protected by this set. This will help to detect .1 files that are produced when - # repairing this kind of set. + # Find extra name matched files + for f in self.protectedFiles: + baseName = utils.getBasename(f) + matches = utils.findFileNameMatches(self.directory, baseName) - # Find "normal" name matched files - self.name_matched_files = rsutil.common.find_name_matches (self.dir, self.basename) + self.similarlyNamedFiles += matches - # Find "extra" name matched files - for f in self.prot_matched_files: - f_basename = rsutil.common.get_basename (f) - f_matches = rsutil.common.find_name_matches (self.dir, f_basename) + # Remove all duplicates + self.similarlyNamedFiles = list(set(self.similarlyNamedFiles)) - self.name_matched_files = rsutil.common.no_duplicates (self.name_matched_files + - f_matches) + # Update the allFiles set + self.allFiles = set(self.similarlyNamedFiles + self.protectedFiles) - # Update the all_files part now - self.all_files = rsutil.common.no_duplicates (self.name_matched_files + self.prot_matched_files) - - def runAll (self): - """Run the Repair and Deletion stages, omitting the Extraction stage""" - - # Repair Stage - try: - self.runVerifyAndRepair() - except (RuntimeError, OSError): - logging.critical('Repair stage failed for: %s' % self.p2file) - raise - - self.update_matches () - - # Deletion Stage - try: - self.runDelete() - except (RuntimeError, OSError): - logging.critical('Delete stage failed for: %s' % self.p2file) - raise - - logging.info ('Successfully completed: %s' % self.p2file) + ############################################################################ diff --git a/PAR2Set/OldRAR.py b/PAR2Set/OldRAR.py index b139570..b1b219e 100644 --- a/PAR2Set/OldRAR.py +++ b/PAR2Set/OldRAR.py @@ -42,37 +42,23 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import os -import PAR2Set.Base -import rsutil.common +from PAR2Set import Base, utils +class OldRAR(Base): -def detector (name_files, prot_files): - """Detect OldRAR sets""" + ############################################################################ - return rsutil.common.has_a_match ('^.*\.r00$', prot_files) + def detect(self): + if not utils.hasAMatch('^.*\.r00$', self.protectedFiles): + raise TypeError + ############################################################################ -class OldRAR (PAR2Set.Base.Base): + def extract(self): + files = utils.findMatches('^.*\.rar$', self.protectedFiles) - """Class for working with old-style rar sets""" + for f in files: + utils.runCommand(['unrar', 'x', '-o+', f], self.directory) - def __repr__ (self): - return 'OLDRAR' - - def find_extraction_heads (self): - """Find the heads of extraction for an old-style rar set""" - - return rsutil.common.find_matches ('^.*\.rar$', self.all_files) - - def extraction_function (self, file, todir): - """Extract a single rar file to the given directory. - - file -- the file to extract - todir -- the directory to extract the file to""" - - assert os.path.isfile (file) - assert os.path.isdir (todir) - - rsutil.common.run_command(['unrar', 'x', '-o+', file], todir) + ############################################################################ diff --git a/PAR2Set/ZIP.py b/PAR2Set/ZIP.py index 39a83ff..75c8169 100644 --- a/PAR2Set/ZIP.py +++ b/PAR2Set/ZIP.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: +# vim: set ts=4 sts=4 sw=4 textwidth=80: """ Holds the ZIP class. @@ -19,7 +19,7 @@ X, but is not required to be. """ __author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006-2008 Ira W. Snyder (devel@irasnyder.com)" __license__ = "GNU GPL v2 (or, at your option, any later version)" # ZIP.py -- detect and work with zip sets @@ -40,34 +40,30 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import PAR2Set.Base -import rsutil.common +from PAR2Set import Base, utils +class ZIP(Base): -def detector (name_files, prot_files): - """Detect a zip set""" + ############################################################################ - all_files = rsutil.common.no_duplicates (name_files + prot_files) - return rsutil.common.has_a_match ('^.*\.zip$', all_files) + def detect(self): + regex = r'^.*\.zip$' -class ZIP (PAR2Set.Base.Base): + if utils.hasAMatch(regex, self.allFiles): + return - """Class for working with normal zip sets""" + raise TypeError - def __repr__ (self): - return 'ZIP' + ############################################################################ - def find_extraction_heads (self): - """Find the heads of extraction for a zip set""" + def extract(self): - return rsutil.common.find_matches ('^.*\.zip', self.all_files) + regex = r'^.*\.zip$' + files = utils.findMatches(regex, self.allFiles) - def extraction_function (self, file, todir): - """Extract a single zip file to the given directory. + for f in files: + utils.runCommand(['unzip', f], todir) - file -- the file to extract - todir -- the directory to extract into""" - - rsutil.common.run_command(['unzip', file], todir) + ############################################################################ diff --git a/PAR2Set/__init__.py b/PAR2Set/__init__.py index b900b57..aa57931 100644 --- a/PAR2Set/__init__.py +++ b/PAR2Set/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: +# vim: set ts=4 sts=4 sw=4 textwidth=80: """ A package which has many different types of PAR2Sets and their corresponding @@ -10,19 +10,26 @@ extra files from the corresponding set of files. """ __author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006, Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006-2008 Ira W. Snyder (devel@irasnyder.com)" __license__ = "GNU GPL v2 (or, at your option, any later version)" -import Base -import ExtractFirstBase -import ExtractFirstNewRAR -import ExtractFirstOldRAR -import Join -import NewRAR -import OldRAR -import ZIP -import NoExtract +# Utilities and a PAR2 file parser +import utils +import par2parser -__all__ = ['Base', 'ExtractFirstBase', 'ExtractFirstNewRAR', 'ExtractFirstOldRAR', 'Join', - 'NewRAR', 'OldRAR', 'ZIP', 'NoExtract'] +# All of the PAR2Set classes +from CompareSet import CompareSet +from Base import Base +from NewRAR import NewRAR +from OldRAR import OldRAR +from Join import Join +from NoExtract import NoExtract +from ZIP import ZIP +from ExtractFirstBase import ExtractFirstBase +from ExtractFirstNewRAR import ExtractFirstNewRAR +from ExtractFirstOldRAR import ExtractFirstOldRAR + +__all__ = ['utils', 'par2parser', 'CompareSet', 'Base', 'ExtractFirstBase', + 'ExtractFirstNewRAR', 'ExtractFirstOldRAR', 'Join', 'NewRAR', + 'NoExtract', 'OldRAR', 'ZIP'] diff --git a/PAR2Set/par2parser.py b/PAR2Set/par2parser.py new file mode 100644 index 0000000..205b308 --- /dev/null +++ b/PAR2Set/par2parser.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=80: + +# This breaks my convention, but python requires that it be +# at the top of the file ... +from __future__ import with_statement + +""" +Module which holds PAR2 file parsing functions. + +Much of this code was borrowed from the excellent cfv project. +See http://cfv.sourceforge.net/ for a copy. +""" + +__author__ = "Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006-2008 Ira W. Snyder (devel@irasnyder.com)" +__license__ = "GNU GPL v2 (or, at your option, any later version)" + +# par2parser.py -- PAR2 file parsing utility +# +# Copyright (C) 2006-2008 Ira W. Snyder (devel@irasnyder.com) +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +import struct, errno, os, md5 + +################################################################################ + +def chompnulls(line): + p = line.find('\0') + + if p < 0: + return line + else: + return line[:p] + +################################################################################ + +# Get all of the filenames protected by the parityFile +# +# This code was almost copied from the CFV project +# See the header of this file for more information +def getProtectedFiles(directory, parityFile): + + assert os.path.isdir(directory) + assert os.path.isfile(os.path.join(directory, parityFile)) + + with open(os.path.join(directory, parityFile), 'rb') as file: + + # We always want to do crc checks + docrcchecks = True + + pkt_header_fmt = '< 8s Q 16s 16s 16s' + pkt_header_size = struct.calcsize(pkt_header_fmt) + file_pkt_fmt = '< 16s 16s 16s Q' + file_pkt_size = struct.calcsize(file_pkt_fmt) + main_pkt_fmt = '< Q I' + main_pkt_size = struct.calcsize(main_pkt_fmt) + + seen_file_ids = {} + expected_file_ids = None + filenames = [] + + while True: + d = file.read(pkt_header_size) + if not d: + break + + magic, pkt_len, pkt_md5, set_id, pkt_type = struct.unpack(pkt_header_fmt, d) + + if docrcchecks: + control_md5 = md5.new() + control_md5.update(d[0x20:]) + d = file.read(pkt_len - pkt_header_size) + control_md5.update(d) + + if control_md5.digest() != pkt_md5: + raise EnvironmentError, (errno.EINVAL, \ + "corrupt par2 file - bad packet hash") + + if pkt_type == 'PAR 2.0\0FileDesc': + if not docrcchecks: + d = file.read(pkt_len - pkt_header_size) + + file_id, file_md5, file_md5_16k, file_size = \ + struct.unpack(file_pkt_fmt, d[:file_pkt_size]) + + if seen_file_ids.get(file_id) is None: + seen_file_ids[file_id] = 1 + filename = chompnulls(d[file_pkt_size:]) + filenames.append(filename) + + elif pkt_type == "PAR 2.0\0Main\0\0\0\0": + if not docrcchecks: + d = file.read(pkt_len - pkt_header_size) + + if expected_file_ids is None: + expected_file_ids = [] + slice_size, num_files = struct.unpack(main_pkt_fmt, d[:main_pkt_size]) + num_nonrecovery = (len(d)-main_pkt_size)/16 - num_files + + for i in range(main_pkt_size,main_pkt_size+(num_files+num_nonrecovery)*16,16): + expected_file_ids.append(d[i:i+16]) + + else: + if not docrcchecks: + file.seek(pkt_len - pkt_header_size, 1) + + if expected_file_ids is None: + raise EnvironmentError, (errno.EINVAL, \ + "corrupt or unsupported par2 file - no main packet found") + + for id in expected_file_ids: + if not seen_file_ids.has_key(id): + raise EnvironmentError, (errno.EINVAL, \ + "corrupt or unsupported par2 file - " \ + "expected file description packet not found") + + return filenames + +################################################################################ + +def main(): + pass + +################################################################################ + +if __name__ == '__main__': + main() + diff --git a/PAR2Set/utils.py b/PAR2Set/utils.py new file mode 100644 index 0000000..8cd9408 --- /dev/null +++ b/PAR2Set/utils.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# vim: set ts=4 sts=4 sw=4 textwidth=80: + +""" +Module holding some utilities for use in the PAR2Set classes +""" + +__author__ = "Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2008, Ira W. Snyder (devel@irasnyder.com)" +__license__ = "GNU GPL v2 (or, at your option, any later version)" + +# utilities.py -- some common utilities for use in PAR2Set classes +# +# Copyright (C) 2008 Ira W. Snyder (devel@irasnyder.com) +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software + +import os, re, logging, subprocess + +################################################################################ + +# Strip most types of endings from a filename +def getBasename(fileName): + + # This regular expression should do pretty good at stripping all of the + # common suffixes of of the files that rarslave is intended to work with + regex = r'^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$' + r = re.compile (regex, re.IGNORECASE) + + # Strip off the suffixes one at a time until + while True: + match = r.match(fileName) + + if match == None: + break + + fileName = match.groups()[0] + + # We've stripped everything, return the baseName + return fileName + +################################################################################ + +# Find all of the files in the given directory that have baseName at the +# beginning of their name +def findFileNameMatches(directory, baseName): + + ename = re.escape(baseName) + regex = re.compile(r'^%s.*$' % ename) + files = os.listdir(directory) + + return [f for f in files if regex.match(f)] + +################################################################################ + +def findMatches(regex, iterateable, ignoreCase=True): + + if ignoreCase: + compiledRegex = re.compile(regex, re.IGNORECASE) + else: + compiledRegex = re.compile(regex) + + return [e for e in iterateable if compiledRegex.match(e)] + +################################################################################ + +def hasAMatch(regex, iterateable, ignoreCase=True): + + if ignoreCase: + compiledRegex = re.compile(regex, re.IGNORECASE) + else: + compiledRegex = re.compile(regex) + + for e in iterateable: + if compiledRegex.match(e): + return True + + return False + +################################################################################ + +# Run the specified command-list in the given directory +# @cmd a list formatted for the subprocess module +# @directory the directory in which to run the command +# @return the status code returned by the command +# +# Exceptions: +# subprocess.CalledProcessError when the called process return code is not 0 +def runCommand(cmd, directory): + + logging.debug('===== BEGIN runCommand() DEBUG =====') + logging.debug('Directory: %s' % directory) + logging.debug('Command: %s'% cmd[0]) + for arg in cmd[1:]: + logging.debug('-> %s' % arg) + logging.debug('===== END runCommand() DEBUG =====') + + return subprocess.check_call(cmd, cwd=directory) + +################################################################################ + +# Return the canonical absolute path from a relative path +def absolutePath(path): + return os.path.abspath(os.path.expanduser(path)) + +################################################################################ + diff --git a/RarslaveDetector.py b/RarslaveDetector.py deleted file mode 100644 index dfb2768..0000000 --- a/RarslaveDetector.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: - -""" -Class which runs each detector in the PAR2Set classes, and attempts -to run all matches on the current set. -""" - -__author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006, Ira W. Snyder (devel@irasnyder.com)" -__license__ = "GNU GPL v2 (or, at your option, any later version)" - -# RarslaveDetector.py -# -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import rsutil.common - -# PAR2Set-derived types -import PAR2Set.Join -import PAR2Set.ZIP -import PAR2Set.OldRAR -import PAR2Set.NewRAR -import PAR2Set.ExtractFirstOldRAR -import PAR2Set.ExtractFirstNewRAR -import PAR2Set.NoExtract - -import logging - -class RarslaveDetector (object): - - """A class to detect the type of a set, and then run the appropriate class - on the set.""" - - # A tuple of tuples of the following type: - # (type_detection_function, type_working_class) - TYPES = ( (PAR2Set.Join.detector, PAR2Set.Join.Join), - (PAR2Set.ZIP.detector, PAR2Set.ZIP.ZIP), - (PAR2Set.OldRAR.detector, PAR2Set.OldRAR.OldRAR), - (PAR2Set.NewRAR.detector, PAR2Set.NewRAR.NewRAR), - (PAR2Set.ExtractFirstOldRAR.detector, PAR2Set.ExtractFirstOldRAR.ExtractFirstOldRAR), - (PAR2Set.ExtractFirstNewRAR.detector, PAR2Set.ExtractFirstNewRAR.ExtractFirstNewRAR), - (PAR2Set.NoExtract.detector, PAR2Set.NoExtract.NoExtract), - ) - - def __init__ (self, dir, p2file): - """Constructor - - dir -- the directory containing this set - p2file -- a single PAR2 file from this set""" - - # The real "meat" of the class - self.dir = dir - self.p2file = p2file - self.basename = rsutil.common.get_basename (p2file) - - # Find files that match by name only - self.name_matched_files = rsutil.common.find_name_matches (self.dir, self.basename) - - # Find all par2 files for this set using name matches - self.all_p2files = rsutil.common.find_par2_files (self.name_matched_files) - - # Try to get the protected files for this set - self.prot_matched_files = rsutil.common.parse_all_par2 (self.dir, self.p2file, self.all_p2files) - - def runMatchingTypes (self): - """Run all matching PAR2Set types for which the detection function detects that - the class is valid.""" - - detected = False - success = False - - for (detector, classname) in self.TYPES: - if detector (self.name_matched_files, self.prot_matched_files): - # The detector matched, so we're up and running! - - p2set = classname (self.dir, self.p2file) - - detected = True - logging.debug ('Detected type: %s' % p2set) - - # Try to have rarslave do it's thing - try: - # Have rarslave do it's thing - p2set.runAll () - - # It succeeded, exit the loop early - success = True - break - except (OSError, RuntimeError): - logging.error('Detected type (%s) failed for: %s' % (p2set, self.p2file)) - except: - logging.error('Unknown exception occurred') - raise - - # Make sure we detected at least one valid type - if not detected: - logging.warning ('Unable to determine type: %s' % self.p2file) - logging.debug ('The following information will help in writing a detector:') - logging.debug ('name_matches: %s' % self.name_matched_files) - logging.debug ('prot_matches: %s' % self.prot_matched_files) - - # Make sure that something worked - if success == False: - logging.critical('All types failed for: %s' % self.p2file) - -def main (): - pass - -if __name__ == '__main__': - main () - diff --git a/rarslave-test.py b/rarslave-test.py deleted file mode 100644 index 5790681..0000000 --- a/rarslave-test.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92 : - -import os, sys, unittest -from rarslave import * -from RarslaveLogger import RarslaveLogger,RarslaveMessage,MessageType - -class rarslavetest (unittest.TestCase): - - def assertListEquals (self, l1, l2): - if l1 is l2: - return True - - self.assertEquals (len(l1), len(l2)) - - for e in l1: - if e not in l2: - self.fail ("Element missing from list") - - return True - - def setUp (self): - self.logger = RarslaveLogger () - - def tearDown (self): - self.logger = None - - def testGetBasenameNone (self): - QNAME = '[something] some names.txt' - ANAME = '[something] some names.txt' - - # Tests for an exension that should not be stripped - self.assertEquals (ANAME, get_basename (QNAME)) - - def testGetBasenameSingle (self): - QNAME = '[something] some names.par2' - ANAME = '[something] some names' - - self.assertEquals (ANAME, get_basename (QNAME)) - - def testGetBasenameMulti (self): - QNAME = '[a.f.k.] The Melancholy of Haruhi Suzumiya - 13.avi.001' - ANAME = '[a.f.k.] The Melancholy of Haruhi Suzumiya - 13' - - self.assertEquals (ANAME, get_basename (QNAME)) - - def testGetBasenameMulti2 (self): - QNAME = '[AonE-AnY]_Ah_My_Goddess_-_Sorezore_no_Tsubasa_-_13_[WS][E6380C3F].avi.vol00+01.PAR2' - ANAME = '[AonE-AnY]_Ah_My_Goddess_-_Sorezore_no_Tsubasa_-_13_[WS][E6380C3F]' - - self.assertEquals (ANAME, get_basename (QNAME)) - - def testFindLikelyFilesBadDir (self): - DIR = '/fake/dir' - - self.assertRaises (AssertionError, find_likely_files, "fake", DIR) - - def testFindAllPar2Files (self): - DIR = '/fake/dir' - - self.assertRaises (ValueError, find_all_par2_files, DIR) - - def testIsNewRar (self): - DIR = os.getcwd() + '/test_material/01/' - - self.assertTrue (is_newrar (os.listdir (DIR))) - - def testDeletableFiles1 (self): - FILES = ['test.part%d.rar' % n for n in xrange(10)] - - self.assertListEquals (find_deleteable_files (FILES), FILES) - - def testDeletableFiles2 (self): - FILESN = ['%d.mp3' % n for n in xrange(20)] - FILESY = ['%d.zip' % n for n in xrange(5)] - - self.assertListEquals (find_deleteable_files (FILESN + FILESY), FILESY) - - ### RarslaveMessage tests - - def testRepr (self): - STR1 = "Hello World" - STR2 = "Goodbye, \nCruel World" - - self.assertEquals (STR1, RarslaveMessage (STR1).__repr__()) - self.assertEquals (STR1, RarslaveMessage (STR1, MessageType.Normal).__repr__()) - self.assertEquals (STR2, RarslaveMessage (STR2, MessageType.Verbose).__repr__()) - self.assertEquals (STR2, RarslaveMessage (STR2, MessageType.Debug).__repr__()) - - def testisVerboseMessage (self): - STR1 = "Hello World" - STR2 = "Goodbye, \nCruel World" - - self.assertTrue (RarslaveMessage (STR1, MessageType.Verbose).isVerbose()) - self.assertTrue (RarslaveMessage (STR2, MessageType.Verbose).isVerbose()) - self.assertFalse (RarslaveMessage (STR1).isVerbose()) - self.assertFalse (RarslaveMessage (STR2, MessageType.Debug).isVerbose()) - self.assertFalse (RarslaveMessage (STR2, MessageType.Normal).isVerbose()) - - -if __name__ == '__main__': - unittest.main () - diff --git a/rarslave.py b/rarslave.py index 18bb540..edd34ce 100755 --- a/rarslave.py +++ b/rarslave.py @@ -1,21 +1,20 @@ #!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: +# vim: set ts=4 sts=4 sw=4 textwidth=80: """ -The main program of the rarslave project. +The main program of the rarslave project -This handles all of the commandline, configuration file, and option -work. It gets the environment set up for a run using the RarslaveDetector -class. +This handles all of the commandline and configuration file work, then tries to +repair, extract, and delete any PAR2Sets that it finds. """ __author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" +__copyright__ = "Copyright (c) 2006-2008 Ira W. Snyder (devel@irasnyder.com)" __license__ = "GNU GPL v2 (or, at your option, any later version)" # rarslave.py -- a usenet autorepair and autoextract utility # -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) +# Copyright (C) 2006-2008 Ira W. Snyder (devel@irasnyder.com) # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,268 +30,334 @@ __license__ = "GNU GPL v2 (or, at your option, any later version)" # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -VERSION="2.0.0" -PROGRAM="rarslave" +VERSION = "2.1.0" +PROGRAM = "rarslave" -import os, sys, optparse, logging -import rsutil -import RarslaveDetector +import os, sys, optparse, logging, ConfigParser +from subprocess import CalledProcessError +import PAR2Set -# Global options from the rsutil.globals class -options = rsutil.globals.options -config = rsutil.globals.config +################################################################################ -# A tiny class to hold logging output until we're finished -class DelayedLogger (object): +# A simple-ish configuration class +class RarslaveConfig(object): - """A small class to hold logging output until the program is finished running. - It emulates sys.stdout in the needed ways for the logging module.""" + DEFAULT_CONFIG_FILE = PAR2Set.utils.absolutePath( + os.path.join('~', '.config', 'rarslave2', 'rarslave2.conf')) - def __init__ (self, output=sys.stdout.write): - self.__messages = [] - self.__output = output + def __init__(self, fileName=DEFAULT_CONFIG_FILE): - def write (self, msg): - self.__messages.append (msg) + # Make sure that the fileName is in absolute form + self.fileName = os.path.abspath(os.path.expanduser(fileName)) - def flush (self): - pass + # Open it with ConfigParser + self.config = ConfigParser.SafeConfigParser() + self.config.read(fileName) - def size (self): - """Returns the number of messages queued for printing""" - return len (self.__messages) + # Setup the default dictionary + self.defaults = dict() - def close (self): - """Print all messages, clear the queue""" - for m in self.__messages: - self.__output (m) + # Add all of the defaults + self.add_default('directories', 'start', + os.path.join('~', 'downloads'), + PAR2Set.utils.absolutePath) + self.add_default('options', 'recursive', True, self.toBool) + self.add_default('options', 'interactive', False, self.toBool) + self.add_default('options', 'verbosity', 0, self.toInt) + self.add_default('options', 'delete', True, self.toBool) - self.__messages = [] + # Add a new default value + def add_default(self, section, key, value, typeConverter): -# A tiny class used to find unique PAR2 sets -class CompareSet (object): + self.defaults[(section, key)] = (value, typeConverter) - """A small class used to find unique PAR2 sets""" + # Get the default value + def get_default(self, section, key): - def __init__ (self, dir, p2file): - self.dir = dir - self.p2file = p2file + (value, typeConverter) = self.defaults[(section, key)] + return value - self.basename = rsutil.common.get_basename (self.p2file) - self.name_matches = rsutil.common.find_name_matches (self.dir, self.basename) + # Coerce the value from a string into the correct type + def coerceValue(self, section, key, value): - def __eq__ (self, rhs): - return (self.dir == rhs.dir) \ - and (self.basename == rhs.basename) \ - and rsutil.common.list_eq (self.name_matches, rhs.name_matches) + (defaultValue, typeConverter) = self.defaults[(section, key)] + # Try the coercion, error and exit if there is a problem + try: + return typeConverter(value) + except: + sys.stderr.write('Unable to parse configuration file\n') + sys.stderr.write('-> at section: %s\n' % section) + sys.stderr.write('-> at key: %s\n' % key) + sys.exit(2) -def find_all_par2_files (dir): - """Finds all par2 files in the given directory. + # Return the value + def get(self, section, key): - dir -- the directory in which to search for PAR2 files + try: + # Get the user-provided value + value = self.config.get(section, key) + except: + # Oops, they didn't provide it, use the default + # NOTE: if you get an exception here, check your code ;) + value = self.defaults[(section, key)] - NOTE: does not return absolute paths""" + # Try to evaluate some safe things, for convenience + return self.coerceValue(section, key, value) - if not os.path.isdir (os.path.abspath (dir)): - raise ValueError # bad directory given + # Convert a string to an int (any base) + def toInt(s): + return int(s, 0) - dir = os.path.abspath (dir) - files = os.listdir (dir) + # Mark it static + toInt = staticmethod(toInt) - return rsutil.common.find_par2_files (files) + # Convert a string to a bool + def toBool(s): + if s in ['t', 'T', 'True', 'true', 'yes', '1']: + return True -def generate_all_parsets (dir): - """Generate all parsets in the given directory + if s in ['f', 'F', 'False', 'false', 'no', '0']: + return False - dir -- the directory in which to search""" + raise ValueError - assert os.path.isdir (dir) # Directory MUST be valid + # Mark it static + toBool = staticmethod(toBool) - parsets = [] - p2files = find_all_par2_files (dir) +################################################################################ - for f in p2files: - p = CompareSet (dir, f) - if p not in parsets: - parsets.append (p) +# Global configuration, read from default configuration file +config = RarslaveConfig() - return [(p.dir, p.p2file) for p in parsets] +################################################################################ -def check_required_progs(): - """Check if the required programs are installed""" +# A tiny class to hold logging output until we're finished +class DelayedLogger (object): - needed = [] + """A small class to hold logging output until the program is finished running. + It emulates sys.stdout in the needed ways for the logging module.""" - try: - rsutil.common.run_command(['par2repair', '--help']) - except OSError: - needed.append('par2repair') - except RuntimeError: - pass + def __init__ (self, output=sys.stdout.write): + self.__messages = [] + self.__output = output - try: - rsutil.common.run_command(['unrar', '--help']) - except OSError: - needed.append('unrar') - except RuntimeError: - pass + def write (self, msg): + self.__messages.append (msg) - try: - rsutil.common.run_command(['unzip', '--help']) - except OSError: - needed.append('unzip') - except RuntimeError: - pass + def flush (self): + pass - if needed: - for n in needed: - print 'Needed program "%s" not found in $PATH' % (n, ) + def size (self): + """Returns the number of messages queued for printing""" + return len (self.__messages) - sys.exit(1) + def close (self): + """Print all messages, clear the queue""" + map(self.__output, self.__messages) + self.__messages = [] -def run_options (options): - """Process all of the commandline options, doing thing such as printing the - version number, etc.""" +################################################################################ - # Fix directories - options.work_dir = rsutil.common.full_abspath (options.work_dir) +# Convert from the verbose command line option to the logging level that +# will be used by the logging class to print messages +def findLogLevel(options): - # Make sure that the directory is valid - if not os.path.isdir (options.work_dir): - sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.work_dir) - sys.stderr.write ('option to override the working directory temporarily, or edit the\n') - sys.stderr.write ('configuration file to override the working directory permanently.\n') - sys.exit (1) + level = options.verbose - options.quiet - if options.extract_dir != None: - options.extract_dir = rsutil.common.full_abspath (options.extract_dir) + if level < -3: + level = -3 - if options.version: - print PROGRAM + ' - ' + VERSION - print - print 'Copyright (c) 2005,2006 Ira W. Snyder (devel@irasnyder.com)' - print - print 'This program comes with ABSOLUTELY NO WARRANTY.' - print 'This is free software, and you are welcome to redistribute it' - print 'under certain conditions. See the file COPYING for details.' - sys.exit (0) + if level > 1: + level = 1 - if options.check_progs: - check_required_progs () - sys.exit (0) + LEVELS = { + 1 : logging.DEBUG, + 0 : logging.INFO, + -1 : logging.WARNING, + -2 : logging.ERROR, + -3 : logging.CRITICAL + } - if options.write_def_config: - config.write_config (default=True) - sys.exit (0) + return LEVELS[level] - if options.write_config: - config.write_config () - sys.exit (0) +################################################################################ -def find_loglevel (options): - """Find the log level that should be printed by the logging class""" +def parseCommandLineOptions(): - loglevel = options.verbose - options.quiet + # Build the OptionParser + parser = optparse.OptionParser() + parser.add_option('-n', '--not-recursive', dest='recursive', action='store_false', + default=config.get('options', 'recursive'), + help="Don't run recursively") - if loglevel > 1: - loglevel = 1 + parser.add_option('-d', '--directory', dest='directory', type='string', + default=config.get('directories', 'start'), + help="Start working at DIR", metavar='DIR') - if loglevel < -3: - loglevel = -3 + parser.add_option('-i', '--interactive', dest='interactive', action='store_true', + default=config.get('options', 'interactive'), + help="Confirm before removing files") - LEVELS = { 1 : logging.DEBUG, - 0 : logging.INFO, - -1: logging.WARNING, - -2: logging.ERROR, - -3: logging.CRITICAL - } + parser.add_option('--no-delete', dest='delete', action='store_false', + default=config.get('options', 'delete'), + help="Do not delete files used to repair") - return LEVELS [loglevel] + parser.add_option('-q', '--quiet', dest='quiet', action='count', + default=0, help="Output fatal messages only") -def main (): + parser.add_option('-v', '--verbose', dest='verbose', action='count', + default=config.get('options', 'verbosity'), + help="Output extra information") + + parser.add_option('-V', '--version', dest='version', action='store_true', + default=False, help="Output version information") + + parser.version = VERSION + + # Parse the given options + (options, args) = parser.parse_args() + + # Postprocess the options, basically sanitizing them + options.directory = PAR2Set.utils.absolutePath(options.directory) - # Setup the logger - logger = DelayedLogger () - logging.basicConfig (stream=logger, level=logging.WARNING, \ - format='%(levelname)-8s %(message)s') + # Make sure that the directory is valid + if not os.path.isdir (options.directory): + sys.stderr.write ('\"%s\" is not a valid directory. Use the \"-d\"\n' % options.directory) + sys.stderr.write ('option to override the working directory temporarily, or edit the\n') + sys.stderr.write ('configuration file to override the working directory permanently.\n') + sys.exit (1) - # Build the OptionParser - parser = optparse.OptionParser() - parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive', - default=rsutil.common.config_get_value('options', 'recursive'), - help="Don't run recursively") + if options.version: + print PROGRAM + ' - ' + VERSION + print + print 'Copyright (c) 2005-2008 Ira W. Snyder (devel@irasnyder.com)' + print + print 'This program comes with ABSOLUTELY NO WARRANTY.' + print 'This is free software, and you are welcome to redistribute it' + print 'under certain conditions. See the file COPYING for details.' + sys.exit (0) - parser.add_option('-d', '--work-dir', dest='work_dir', type='string', - default=rsutil.common.config_get_value('directories', 'working_directory'), - help="Start running at DIR", metavar='DIR') + return (options, args) - parser.add_option('-e', '--extract-dir', dest='extract_dir', type='string', - default=rsutil.common.config_get_value('directories', 'extract_directory'), - help="Extract to DIR", metavar='DIR') +################################################################################ - parser.add_option('-p', '--check-required-programs', - action='store_true', dest='check_progs', - default=False, - help="Check for required programs") +# Find each unique CompareSet in the given directory and set of files +def findUniqueSets(directory, files): - parser.add_option('-f', '--write-default-config', - action='store_true', dest='write_def_config', - default=False, help="Write out a new default config") + regex = r'^.*\.par2' + s = [] - parser.add_option('-c', '--write-new-config', - action='store_true', dest='write_config', - default=False, help="Write out the current config") + for f in PAR2Set.utils.findMatches(regex, files): - parser.add_option('-i', '--interactive', dest='interactive', action='store_true', - default=rsutil.common.config_get_value('options', 'interactive'), - help="Confirm before removing files") + try: + c = PAR2Set.CompareSet(directory, f) - parser.add_option('-q', '--quiet', dest='quiet', action='count', - default=0, help="Output fatal messages only") + if c not in s: + s.append(c) + except: + # We just ignore any errors that happen, such as + # parsing the PAR file + pass - parser.add_option('-v', '--verbose', dest='verbose', action='count', - default=0, help="Output extra information") + return s - parser.add_option('-V', '--version', dest='version', action='store_true', - default=False, help="Output version information") +################################################################################ - parser.version = VERSION +# Run each PAR2Set type on a CompareSet +def runEachType(cs, options): - # Parse the given options - global options - (rsutil.globals.options, args) = parser.parse_args() - options = rsutil.globals.options + types = ( + PAR2Set.Join, + PAR2Set.ZIP, + PAR2Set.OldRAR, + PAR2Set.NewRAR, + PAR2Set.ExtractFirstOldRAR, + PAR2Set.ExtractFirstNewRAR, + PAR2Set.NoExtract, + ) + + detected = False + + # Try to detect each type in turn + for t in types: + try: + instance = t(cs, options) + detected = True + logging.debug('%s detected for %s' % (t.__name__, cs.parityFile)) + except TypeError: + logging.debug('%s not detected for %s' % (t.__name__, cs.parityFile)) + continue + + # We detected something, try to run it + try: + instance.run() + logging.info('Success: %s' % instance) + + # Leave early, we're done + return + except (OSError, CalledProcessError): + logging.critical('Failure: %s' % instance) + + # Check that at least one detection worked + if not detected: + logging.critical('Detection failed: %s' % cs.parityFile) + logging.debug('The following information will help to create a detector') + logging.debug('===== BEGIN CompareSet RAW INFO =====') + logging.debug(str(cs)) + logging.debug('===== END CompareSet RAW INFO =====') + + # If we got here, either the detection didn't work or the run itself didn't + # work, so print out the message telling the user that we were unsuccessful + logging.critical('Unsuccessful: %s' % cs.parityFile) + +################################################################################ + +def runDirectory(directory, files, options): + + logging.debug('Running in directory: %s' % directory) + sets = findUniqueSets(directory, files) + + for cs in sets: + try: + runEachType(cs, options) + except: + logging.error('Unknown Exception: %s' % cs.parityFile) + +################################################################################ + +def main (): - # Run any special actions that are needed on these options - run_options (options) + # Parse all of the command line options + (options, args) = parseCommandLineOptions() - # Find the loglevel using the options given - logging.getLogger().setLevel (find_loglevel (options)) + # Set up the logger + logger = DelayedLogger() + logging.basicConfig(stream=logger, level=logging.WARNING, \ + format='%(levelname)-8s %(message)s') + logging.getLogger().setLevel (findLogLevel(options)) - # Run recursively - if options.recursive: - for (dir, subdirs, files) in os.walk (options.work_dir): - parsets = generate_all_parsets (dir) - for (p2dir, p2file) in parsets: - detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) - detector.runMatchingTypes () + # Run recursively + if options.recursive: + for (directory, subDirectories, files) in os.walk(options.directory): + runDirectory(directory, files, options) - # Non-recursive - else: - parsets = generate_all_parsets (options.work_dir) - for (p2dir, p2file) in parsets: - detector = RarslaveDetector.RarslaveDetector (p2dir, p2file) - detector.runMatchingTypes () + # Non-recursive + else: + directory = options.directory + files = os.listdir(directory) - # Print the results - if logger.size () > 0: - print '\nLog\n' + '=' * 80 - logger.close () + runDirectory(directory, files, options) - # Done! - return 0 + # Print out all of the messages that have been accumulating + # in the DelayedLogger() + if logger.size() > 0: + print + print 'Log' + print '=' * 80 + logger.close() +# Check if we were called directly if __name__ == '__main__': - main () + main () diff --git a/rsutil/__init__.py b/rsutil/__init__.py deleted file mode 100644 index 73c4551..0000000 --- a/rsutil/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import globals -import common -import config -import par2parser - -__all__ = ['common', 'globals', 'config', 'par2parser'] - diff --git a/rsutil/common.py b/rsutil/common.py deleted file mode 100644 index 370d7d6..0000000 --- a/rsutil/common.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: - -""" -Module holding all of the common functions used throughout the rarslave project -""" - -__author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006, Ira W. Snyder (devel@irasnyder.com)" -__license__ = "GNU GPL v2 (or, at your option, any later version)" - -# common.py -- holds all of the common functions used throughout the rarslave project -# -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software - -import os -import re -import logging -import subprocess -import exceptions - -import rsutil.globals -import rsutil.par2parser - -def find_matches (regex, li, ignorecase=True): - - if ignorecase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - return [e for e in li if cregex.match (e)] - -def has_a_match (regex, li, ignorecase=True): - - if ignorecase: - cregex = re.compile (regex, re.IGNORECASE) - else: - cregex = re.compile (regex) - - for e in li: - if cregex.match (e): - return True - - return False - -def no_duplicates (li): - """Removes all duplicates from a list""" - return list(set(li)) - -def run_command (cmd, indir=None): - # Runs the specified command-line in the directory given (or, in the current directory - # if none is given). It returns the status code given by the application. - - if indir == None: - indir = os.getcwd() - else: - assert os.path.isdir (indir) # MUST be a directory! - - print 'RUNNING COMMAND' - print 'Directory: %s' % indir - print 'Command: %s' % cmd[0] - for f in cmd[1:]: - print '-> %s' % f - - ret = subprocess.Popen(cmd, cwd=indir).wait() - - if ret != 0: - raise RuntimeError - - return ret - -def full_abspath (p): - return os.path.abspath (os.path.expanduser (p)) - -def find_par2_files (files): - """Find all par2 files in the list $files""" - - PAR2_REGEX = config_get_value ('regular expressions', 'par2_regex') - regex = re.compile (PAR2_REGEX, re.IGNORECASE) - return [f for f in files if regex.match (f)] - -def find_name_matches (dir, basename): - """Finds files which are likely to be part of the set corresponding - to $name in the directory $dir""" - - assert os.path.isdir (dir) - - ename = re.escape (basename) - regex = re.compile ('^%s.*$' % (ename, )) - - return [f for f in os.listdir (dir) if regex.match (f)] - -def parse_all_par2 (dir, p2head, p2files): - """Searches though p2files and tries to parse at least one of them""" - done = False - files = [] - - for f in p2files: - - # Exit early if we've found a good file - if done: - break - - try: - files = rsutil.par2parser.get_protected_files (dir, f) - done = True - except (EnvironmentError, OSError, OverflowError): - logging.warning ('Corrupt PAR2 file: %s' % f) - - # Now that we're out of the loop, check if we really finished - if not done: - logging.critical ('All PAR2 files corrupt for: %s' % p2head) - - # Return whatever we've got, empty or not - return files - -def get_basename (name): - """Strips most kinds of endings from a filename""" - - regex = config_get_value ('regular expressions', 'basename_regex') - r = re.compile (regex, re.IGNORECASE) - done = False - - while not done: - done = True - - if r.match (name): - g = r.match (name).groups() - name = g[0] - done = False - - return name - -def list_eq (l1, l2): - - if len(l1) != len(l2): - return False - - return set(l1) == set(l2) - -# Convience functions for the config - -def config_get_value (section, name): - return rsutil.globals.config.get_value (section, name) - -# Convience functions for the options - -def options_get_value (name): - return getattr (rsutil.globals.options, name) - diff --git a/rsutil/config.py b/rsutil/config.py deleted file mode 100644 index bc8c58e..0000000 --- a/rsutil/config.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: - -""" -Module holding the config class, to be used by the rarslave project. -""" - -__author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" -__license__ = "GNU GPL v2 (or, at your option, any later version)" - -# config.py -- configuration class for rarslave -# -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import os, ConfigParser - -class config (object): - """A simple class to hold the default configs for the whole program""" - - DEFAULT_CONFIG=os.path.join ('~','.config','rarslave2','rarslave2.conf') - - def __read_config(self, filename=DEFAULT_CONFIG): - """Attempt to open and read the rarslave config file""" - - # Make sure the filename is corrected - filename = os.path.abspath(os.path.expanduser(filename)) - - user_config = {} - - # Write the default config if it doesn't exist - if not os.path.isfile(filename): - self.write_config(default=True) - - config = ConfigParser.ConfigParser() - config.read(filename) - - for section in config.sections(): - for option in config.options(section): - user_config[(section, option)] = config.get(section, option) - - return user_config - - def write_config(self, filename=DEFAULT_CONFIG, default=False): - """Write out the current config to the config file. If you set default=True, then - the default config file will be written.""" - - config = ConfigParser.ConfigParser() - - # Correct filename - filename = os.path.abspath(os.path.expanduser(filename)) - - # Reset all config to make sure we write the default one, if necessary - if default: - self.__user_config = {} - print 'Writing default config to %s' % (filename, ) - - # [directories] section - config.add_section('directories') - for (s, k) in self.__defaults.keys(): - if s == 'directories': - config.set(s, k, self.get_value(s, k)) - - # [options] section - config.add_section('options') - for (s, k) in self.__defaults.keys(): - if s == 'options': - config.set(s, k, self.get_value(s, k)) - - # [regular_expressions] section - config.add_section('regular expressions') - for (s, k) in self.__defaults.keys(): - if s == 'regular expressions': - config.set(s, k, self.get_value(s, k)) - - # [commands] section - config.add_section('commands') - for (s, k) in self.__defaults.keys(): - if s == 'commands': - config.set(s, k, self.get_value(s, k)) - - # Try to make the ~/.config/rarslave/ directory - if not os.path.isdir(os.path.split(filename)[0]): - try: - os.makedirs(os.path.split(filename)[0]) - except: - print 'Could not make directory: %s' % (os.path.split(filename)[0], ) - sys.exit() - - # Try to write the config file to disk - try: - fsock = open(filename, 'w') - try: - config.write(fsock) - finally: - fsock.close() - except: - print 'Could not open: %s for writing' % (filename, ) - sys.exit() - - def __get_default_val(self, section, key): - return self.__defaults[(section, key)] - - def get_value(self, section, key): - """Get a config value. Attempts to get the value from the user's - config first, and then uses the default.""" - - try: - value = self.__user_config[(section, key)] - except: - # This should work, unless you write something stupid - # into the code, so DON'T DO IT - value = self.__get_default_val(section, key) - - # Convert config options to native types for easier use - SAFE_EVAL = ['None', 'True', 'False', '-1', '0', '1', '2'] - - if value in SAFE_EVAL: - value = eval (value) - - # Absolute-ize directories for easier use - if section == 'directories' and value != None: - value = os.path.abspath (os.path.expanduser (value)) - - return value - - def __init__(self): - self.__defaults = { - ('directories', 'working_directory') : os.path.join ('~','downloads','usenet'), - ('directories', 'extract_directory') : None, - ('options', 'recursive') : True, - ('options', 'interactive') : False, - ('options', 'output_loglevel') : 0, - ('regular expressions', 'par2_regex') : '^.*\.par2$', - ('regular expressions', 'delete_regex') : - '^.*\.(par2|\d|\d\d\d|rar|r\d\d|zip)$', - ('regular expressions', 'basename_regex') : - '^(.+)\.(par2|vol\d+\+\d+|\d\d\d|part\d+|rar|zip|avi|mp4|mkv|ogm)$', - ('regular expressions', 'extractable_regex') : - '^.+\.(rar|r\d\d|\d\d\d|zip)$', - ('commands', 'unrar') : 'unrar x -o+ -- ', - ('commands', 'unzip') : 'unzip \"%s\" -d \"%s\" ', - ('commands', 'noextract') : 'mv \"%s\" \"%s\" ', - ('commands', 'par2repair') : 'par2repair -- ', - } - - self.__user_config = self.__read_config() - - - - -def main (): - pass - -if __name__ == '__main__': - main () - diff --git a/rsutil/globals.py b/rsutil/globals.py deleted file mode 100644 index 6287d0d..0000000 --- a/rsutil/globals.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: - -""" -Module which holds global variables needed throughout the rarslave project. -""" - -__author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" -__license__ = "GNU GPL v2 (or, at your option, any later version)" - -# globals.py -- global variable storage for the rarslave project -# -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import rsutil.config - -# I know that this sucks majorly, but I kinda need it to keep things -# sane in the code. I don't /want/ to have to keep passing the config -# and options around all the time. - -# This will hold the configuration from the configuration file. This should -# only be used to hold statically, non-runtime alterable content. -config = rsutil.config.config () - -# This will hold the configuration from the command-line options. You should -# probably be using this to get values in most cases. It takes its defaults -# from the configuration file. -options = None - diff --git a/rsutil/par2parser.py b/rsutil/par2parser.py deleted file mode 100644 index 572c7b1..0000000 --- a/rsutil/par2parser.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sts=4 sw=4 textwidth=92: - -""" -Module which holds PAR2 file parsing functions. - -Much of this code was borrowed from the excellent cfv project. -See http://cfv.sourceforge.net/ for a copy. -""" - -__author__ = "Ira W. Snyder (devel@irasnyder.com)" -__copyright__ = "Copyright (c) 2006,2007 Ira W. Snyder (devel@irasnyder.com)" -__license__ = "GNU GPL v2 (or, at your option, any later version)" - -# par2parser.py -- PAR2 file parsing utility -# -# Copyright (C) 2006,2007 Ira W. Snyder (devel@irasnyder.com) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -import struct, errno, os, md5 - -def chompnulls(line): - p = line.find('\0') - if p < 0: return line - else: return line[:p] - -def get_protected_files (dir, filename): - """Get all of the filenames that are protected by the par2 - file given as the filename""" - - assert os.path.isdir (dir) # MUST be a valid directory - assert os.path.isfile (os.path.join (dir, filename)) - - full_filename = os.path.join (dir, filename) - - try: - file = open(full_filename, 'rb') - except: - print 'Could not open %s' % (full_filename, ) - return [] - - # We always want to do crc checks - docrcchecks = True - - pkt_header_fmt = '< 8s Q 16s 16s 16s' - pkt_header_size = struct.calcsize(pkt_header_fmt) - file_pkt_fmt = '< 16s 16s 16s Q' - file_pkt_size = struct.calcsize(file_pkt_fmt) - main_pkt_fmt = '< Q I' - main_pkt_size = struct.calcsize(main_pkt_fmt) - - seen_file_ids = {} - expected_file_ids = None - filenames = [] - - # This try is here to ensure that we close the open file before - # returning. Since this code was (pretty much) borrowed verbatim - # from the cfv project, I didn't want to refactor it to make file - # closing more sane, so I just used a try / finally clause. - try: - while 1: - d = file.read(pkt_header_size) - if not d: - break - - magic, pkt_len, pkt_md5, set_id, pkt_type = struct.unpack(pkt_header_fmt, d) - - if docrcchecks: - control_md5 = md5.new() - control_md5.update(d[0x20:]) - d = file.read(pkt_len - pkt_header_size) - control_md5.update(d) - - if control_md5.digest() != pkt_md5: - raise EnvironmentError, (errno.EINVAL, \ - "corrupt par2 file - bad packet hash") - - if pkt_type == 'PAR 2.0\0FileDesc': - if not docrcchecks: - d = file.read(pkt_len - pkt_header_size) - - file_id, file_md5, file_md5_16k, file_size = \ - struct.unpack(file_pkt_fmt, d[:file_pkt_size]) - - if seen_file_ids.get(file_id) is None: - seen_file_ids[file_id] = 1 - filename = chompnulls(d[file_pkt_size:]) - filenames.append(filename) - - elif pkt_type == "PAR 2.0\0Main\0\0\0\0": - if not docrcchecks: - d = file.read(pkt_len - pkt_header_size) - - if expected_file_ids is None: - expected_file_ids = [] - slice_size, num_files = struct.unpack(main_pkt_fmt, d[:main_pkt_size]) - num_nonrecovery = (len(d)-main_pkt_size)/16 - num_files - - for i in range(main_pkt_size,main_pkt_size+(num_files+num_nonrecovery)*16,16): - expected_file_ids.append(d[i:i+16]) - - else: - if not docrcchecks: - file.seek(pkt_len - pkt_header_size, 1) - - if expected_file_ids is None: - raise EnvironmentError, (errno.EINVAL, \ - "corrupt or unsupported par2 file - no main packet found") - - for id in expected_file_ids: - if not seen_file_ids.has_key(id): - raise EnvironmentError, (errno.EINVAL, \ - "corrupt or unsupported par2 file - " \ - "expected file description packet not found") - finally: - file.close () - - return filenames - -def main (): - pass - -if __name__ == '__main__': - main () - diff --git a/setup.py b/setup.py index bccef88..3b652f8 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# vim: set ts=8 sts=8 sw=8 textwidth=80: +# vim: set ts=4 sts=4 sw=4 textwidth=80: from distutils.core import setup @@ -20,8 +20,7 @@ setup ( Rarslave is a program that will automatically check, repair, and extract files which are protected by PAR2. The primary reason for its development was to automate the tedious tasks necessary after downloading files from Usenet.""", - packages=['rsutil', 'PAR2Set'], - py_modules=['RarslaveDetector'], + packages=['PAR2Set'], scripts=['rarslave.py'] ) -- 2.25.1