import RarslaveLogger
# Global Variables
-(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT) = range (4)
+(TYPE_OLDRAR, TYPE_NEWRAR, TYPE_ZIP, TYPE_NOEXTRACT, TYPE_UNKNOWN) = range (5)
(SUCCESS, ECHECK, EEXTRACT, EDELETE) = range(4)
config = RarslaveConfig.RarslaveConfig()
logger = RarslaveLogger.RarslaveLogger ()
class RarslaveExtractor (object):
- def __init__ (self, type):
- self.type = type
- self.heads = []
+ type = None
+ heads = []
- def addHead (self, dir, head):
- assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, head))
+ def __init__ (self, dir, p2files, name_files, prot_files):
+
+ self.dir = dir
+ self.p2files = p2files
+ self.name_matched_files = name_files
+ self.prot_matched_files = prot_files
+
+ # Find the type
+ self.type = self.__find_type ()
+
+ logger.addMessage ('Detected set of type: %s' % self, RarslaveLogger.MessageType.Debug)
+
+ # Find the heads
+ self.heads = self.__find_heads ()
+
+ for h in self.heads:
+ logger.addMessage ('Adding extraction head: %s' % h, RarslaveLogger.MessageType.Debug)
+
+ def __repr__ (self):
+ return \
+ { TYPE_OLDRAR : 'Old RAR',
+ TYPE_NEWRAR : 'New RAR',
+ TYPE_ZIP : 'Zip',
+ TYPE_NOEXTRACT : 'No Extract',
+ TYPE_UNKNOWN : 'Unknown' } [self.type]
+
+ def __find_type (self):
+
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ if self.is_oldrar (all_files):
+ return TYPE_OLDRAR
+ elif self.is_newrar (all_files):
+ return TYPE_NEWRAR
+ elif self.is_zip (all_files):
+ return TYPE_ZIP
+ elif self.is_noextract (all_files):
+ return TYPE_NOEXTRACT
- full_head = os.path.join (dir, head)
- logger.addMessage ('Adding extraction head: %s' % full_head, RarslaveLogger.MessageType.Debug)
- self.heads.append (full_head)
+ return TYPE_UNKNOWN
- def extract (self, todir=None):
+ def __generic_find_heads (self, regex, ignorecase=True):
+
+ heads = []
+
+ if ignorecase:
+ cregex = re.compile (regex, re.IGNORECASE)
+ else:
+ cregex = re.compile (regex)
+
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
+ for f in all_files:
+ if cregex.match (f):
+ heads.append (f)
+
+ return heads
+
+ def __find_heads (self):
+
+ if self.type == TYPE_OLDRAR:
+ return self.__generic_find_heads ('^.*\.rar$')
+ elif self.type == TYPE_NEWRAR:
+ return self.__generic_find_heads ('^.*\.part0*1\.rar$')
+ elif self.type == TYPE_ZIP:
+ return self.__generic_find_heads ('^.*\.zip$')
+ elif self.type == TYPE_NOEXTRACT:
+ return self.prot_matched_files
+
+ return []
+
+ def __create_directory (self, dir):
+ if dir == None:
+ return SUCCESS
+
+ if os.path.isdir (dir):
+ return SUCCESS
+
+ try:
+ os.makedirs (dir)
+ logger.addMessage ('Created directory: %s' % dir, RarslaveLogger.MessageType.Verbose)
+ except OSError:
+ logger.addMessage ('FAILED to create directory: %s' % dir, RarslaveLogger.MessageType.Fatal)
+ return -EEXTRACT
+
+ return SUCCESS
+
+ def runExtract (self, todir=None):
# Extract all heads of this set
+ # Extract to the head's dir if we don't care where to extract
+ if todir == None:
+ todir = self.dir
+
# Create the directory $todir if it doesn't exist
- if todir != None and not os.path.isdir (todir):
- logger.addMessage ('Creating directory: %s' % todir, RarslaveLogger.MessageType.Verbose)
- try:
- os.makedirs (todir)
- except OSError:
- logger.addMessage ('FAILED to create directory: %s' % todir, RarslaveLogger.MessageType.Fatal)
- return -EEXTRACT
+ ret = self.__create_directory (todir)
+
+ if ret != SUCCESS:
+ return -EEXTRACT
# Extract all heads
extraction_func = \
{ TYPE_OLDRAR : self.__extract_rar,
TYPE_NEWRAR : self.__extract_rar,
TYPE_ZIP : self.__extract_zip,
- TYPE_NOEXTRACT : self.__extract_noextract }[self.type]
+ TYPE_NOEXTRACT : self.__extract_noextract,
+ TYPE_UNKNOWN : self.__extract_unknown }[self.type]
# Call the extraction function on each head
for h in self.heads:
- if todir == None:
- # Run in the head's directory
- ret = extraction_func (h, os.path.dirname (h))
- else:
- ret = extraction_func (h, todir)
-
+ full_head = full_abspath (h)
+ ret = extraction_func (full_head, todir)
logger.addMessage ('Extraction Function returned: %d' % ret, RarslaveLogger.MessageType.Debug)
# Check error code
return SUCCESS
+ def __extract_unknown (self, file, todir):
+ return SUCCESS
+ def __generic_matcher (self, files, regex, nocase=False):
+ """Run the regex over the files, and see if one matches or not.
+ NOTE: this does not return the matches, just if a match occurred."""
-class RarslaveRepairer (object):
- # Verify (and repair) the set
- # Make sure it worked, otherwise clean up and return failure
-
- def __init__ (self, dir, file, join=False):
- self.dir = dir # the directory containing the par2 file
- self.file = file # the par2 file
- self.join = join # True if the par2 set is 001 002 ...
+ if nocase:
+ cregex = re.compile (regex, re.IGNORECASE)
+ else:
+ cregex = re.compile (regex)
- assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, file))
+ for f in files:
+ if cregex.match (f):
+ return True
- def checkAndRepair (self):
- # Form the command:
- # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES]
- PAR2_CMD = config.get_value ('commands', 'par2repair')
+ return False
- # Get set up
- basename = get_basename (self.file)
- all_files = find_likely_files (self.dir, self.file)
- all_files.sort ()
- par2_files = find_par2_files (all_files)
+ def is_oldrar (self, files):
+ return self.__generic_matcher (files, '^.*\.r00$')
- # assemble the command
- command = "%s \"%s\" " % (PAR2_CMD, self.file)
+ def is_newrar (self, files):
+ return self.__generic_matcher (files, '^.*\.part0*1\.rar$')
- for f in par2_files:
- if f != self.file:
- command += "\"%s\" " % os.path.split (f)[1]
+ def is_zip (self, files):
+ return self.__generic_matcher (files, '^.*\.zip$')
- if self.join:
- for f in all_files:
- if f not in par2_files:
- command += "\"%s\" " % os.path.split (f)[1]
+ def is_noextract (self, files):
+ # Type that needs no extraction.
+ # TODO: Add others ???
+ return self.__generic_matcher (files, '^.*\.001$')
- # run the command
- ret = run_command (command, self.dir)
+class PAR2Set (object):
- # check the result
- if ret != 0:
- logger.addMessage ('PAR2 Check / Repair failed: %s' % self.file, RarslaveLogger.MessageType.Fatal)
- return -ECHECK
+ dir = None
+ p2file = None # The starting par2
+ basename = None # The p2file's basename
+ all_p2files = []
+ name_matched_files = [] # Files that match by basename of the p2file
+ prot_matched_files = [] # Files that match by being protected members
- return SUCCESS
+ def __init__ (self, dir, p2file):
+ assert os.path.isdir (dir)
+ assert os.path.isfile (os.path.join (dir, p2file))
-def run_command (cmd, indir=None):
- # Runs the specified command-line in the directory given (or, in the current directory
- # if none is given). It returns the status code given by the application.
+ self.dir = dir
+ self.p2file = p2file
+ self.basename = self.__get_basename (p2file)
- pwd = os.getcwd ()
+ # Find files that match by name only
+ self.name_matched_files = self.__find_name_matches (self.dir, self.basename)
- if indir != None:
- assert os.path.isdir (indir) # MUST be a directory!
- os.chdir (indir)
+ # Find all par2 files for this set using name matches
+ self.all_p2files = find_par2_files (self.name_matched_files)
- ret = os.system (cmd)
- os.chdir (pwd)
- return ret
+ # Try to get the protected files for this set
+ self.prot_matched_files = self.__parse_all_par2 ()
-def full_abspath (p):
- return os.path.abspath (os.path.expanduser (p))
+ def __list_eq (self, l1, l2):
-def get_basename (name):
- """Strips most kinds of endings from a filename"""
+ if len(l1) != len(l2):
+ return False
- regex = config.get_value ('regular expressions', 'basename_regex')
- r = re.compile (regex, re.IGNORECASE)
- done = False
+ for e in l1:
+ if e not in l2:
+ return False
- while not done:
- done = True
+ return True
- if r.match (name):
- g = r.match (name).groups()
- name = g[0]
- done = False
+ def __eq__ (self, rhs):
+ return (self.dir == rhs.dir) and (self.basename == rhs.basename) and \
+ self.__list_eq (self.name_matched_files, rhs.name_matched_files) and \
+ self.__list_eq (self.prot_matched_files, rhs.prot_matched_files)
- return name
+ def __get_basename (self, name):
+ """Strips most kinds of endings from a filename"""
-def find_likely_files (dir, p2file):
- """Finds files which are likely to be part of the set corresponding
- to $name in the directory $dir"""
+ regex = config.get_value ('regular expressions', 'basename_regex')
+ r = re.compile (regex, re.IGNORECASE)
+ done = False
- assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, p2file))
+ while not done:
+ done = True
- basename = get_basename (p2file)
+ if r.match (name):
+ g = r.match (name).groups()
+ name = g[0]
+ done = False
- dir = os.path.abspath (dir)
- ename = re.escape (basename)
- regex = re.compile ('^%s.*$' % (ename, ))
+ return name
- name_matches = [f for f in os.listdir (dir) if regex.match (f)]
- try:
- parsed_matches = Par2Parser.get_protected_files (dir, p2file)
- except EnvironmentError:
- parsed_matches = []
- logger.addMessage ('Bad par2 file: %s' % p2file, RarslaveLogger.MessageType.Fatal)
+ def __parse_all_par2 (self):
+ """Searches though self.all_p2files and tries to parse at least one of them"""
+ done = False
+ files = []
- return name_matches + parsed_matches
+ for f in self.all_p2files:
-def find_par2_files (files):
- """Find all par2 files in the list $files"""
+ # Exit early if we've found a good file
+ if done:
+ break
- PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex')
- regex = re.compile (PAR2_REGEX, re.IGNORECASE)
- return [f for f in files if regex.match (f)]
+ try:
+ files = Par2Parser.get_protected_files (self.dir, f)
+ done = True
+ except (EnvironmentError, OSError, OverflowError):
+ logger.addMessage ('Corrupt PAR2 file: %s' % f, RarslaveLogger.MessageType.Fatal)
-def find_all_par2_files (dir):
- """Finds all par2 files in a directory"""
- # NOTE: does NOT return absolute paths
+ # Now that we're out of the loop, check if we really finished
+ if not done:
+ logger.addMessage ('All PAR2 files corrupt for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
- if not os.path.isdir (os.path.abspath (dir)):
- raise ValueError # bad directory given
+ # Return whatever we've got, empty or not
+ return files
- dir = os.path.abspath (dir)
- files = os.listdir (dir)
+ def __find_name_matches (self, dir, basename):
+ """Finds files which are likely to be part of the set corresponding
+ to $name in the directory $dir"""
- return find_par2_files (files)
+ assert os.path.isdir (dir)
-def find_extraction_heads (dir, files):
- """Takes a list of possible files and finds likely heads of
- extraction."""
+ ename = re.escape (basename)
+ regex = re.compile ('^%s.*$' % (ename, ))
- # NOTE: perhaps this should happen AFTER repair is
- # NOTE: successful. That way all files would already exist
+ return [f for f in os.listdir (dir) if regex.match (f)]
- # According to various sources online:
- # 1) pre rar-3.0: .rar .r00 .r01 ...
- # 2) post rar-3.0: .part01.rar .part02.rar
- # 3) zip all ver: .zip
+ def __update_name_matches (self):
+ """Updates the self.name_matched_files variable with the most current information.
+ This should be called after the directory contents are likely to change."""
- extractor = None
- p2files = find_par2_files (files)
+ self.name_matched_files = self.__find_name_matches (self.dir, self.basename)
- # Old RAR type, find all files ending in .rar
- if is_oldrar (files):
- extractor = RarslaveExtractor (TYPE_OLDRAR)
- regex = re.compile ('^.*\.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
+ def __is_joinfile (self, filename):
+ regex = re.compile ('^.*\.\d\d\d$', re.IGNORECASE)
+ if regex.match (filename):
+ return True
- if is_newrar (files):
- extractor = RarslaveExtractor (TYPE_NEWRAR)
- regex = re.compile ('^.*\.part0*1.rar$', re.IGNORECASE)
- for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
+ return False
- if is_zip (files):
- extractor = RarslaveExtractor (TYPE_ZIP)
- regex = re.compile ('^.*\.zip$', re.IGNORECASE)
+ def __should_be_joined (self, files):
for f in files:
- if regex.match (f):
- extractor.addHead (dir, f)
-
- if is_noextract (files):
- # Use the Par2 Parser (from cfv) here to find out what files are protected.
- # Since these are not being extracted, they will be mv'd to another directory
- # later.
- extractor = RarslaveExtractor (TYPE_NOEXTRACT)
-
- for f in p2files:
- done = False
- try:
- prot_files = Par2Parser.get_protected_files (dir, f)
- done = True
- except EnvironmentError:
- logger.addMessage ('Error parsing PAR2 file: %s', f)
- continue
-
- if done:
- break
-
- if done:
- for f in prot_files:
- extractor.addHead (dir, f)
- else:
- logger.addMessage ('Error parsing all PAR2 files in this set ...')
-
- # Make sure we found the type
- if extractor == None:
- logger.addMessage ('Not able to find an extractor for this type of set: %s' % p2files[0],
- RarslaveLogger.MessageType.Verbose)
-
- # No-heads here, but it's better than failing completely
- extractor = RarslaveExtractor (TYPE_NOEXTRACT)
+ if self.__is_joinfile (f):
+ return True
- return extractor
-
-def generic_matcher (files, regex, nocase=False):
- """Run the regex over the files, and see if one matches or not.
- NOTE: this does not return the matches, just if a match occurred."""
-
- if nocase:
- cregex = re.compile (regex, re.IGNORECASE)
- else:
- cregex = re.compile (regex)
-
- for f in files:
- if cregex.match (f):
- return True
+ def runCheckAndRepair (self):
+ PAR2_CMD = config.get_value ('commands', 'par2repair')
- return False
+ # Get set up
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+ join = self.__should_be_joined (all_files)
-def is_oldrar (files):
- return generic_matcher (files, '^.*\.r00$')
+ # assemble the command
+ # par2repair -- PAR2 PAR2_EXTRA [JOIN_FILES]
+ command = "%s \"%s\" " % (PAR2_CMD, self.p2file)
-def is_newrar (files):
- return generic_matcher (files, '^.*\.part0*1\.rar$')
+ for f in self.all_p2files:
+ if f != self.p2file:
+ command += "\"%s\" " % os.path.split (f)[1]
-def is_zip (files):
- return generic_matcher (files, '^.*\.zip$')
+ # Only needed when using par2 to join
+ if join:
+ for f in all_files:
+ if self.__is_joinfile (f):
+ command += "\"%s\" " % os.path.split (f)[1]
-def is_noextract (files):
- # Type that needs no extraction.
- # TODO: Add others ???
- return generic_matcher (files, '^.*\.001$')
+ # run the command
+ ret = run_command (command, self.dir)
-def find_deleteable_files (files):
- # Deleteable types regex should come from the config
- dfiles = []
- DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex')
- dregex = re.compile (DELETE_REGEX, re.IGNORECASE)
+ # check the result
+ if ret != 0:
+ logger.addMessage ('PAR2 Check / Repair failed: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
+ return -ECHECK
- return [f for f in files if dregex.match (f)]
+ return SUCCESS
-def printlist (li):
- for f in li:
- print f
+ def __find_deleteable_files (self):
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+ DELETE_REGEX = config.get_value ('regular expressions', 'delete_regex')
+ dregex = re.compile (DELETE_REGEX, re.IGNORECASE)
-class PAR2Set (object):
+ return [f for f in all_files if dregex.match (f)]
- dir = None
- file = None
- likely_files = []
+ def __delete_list_of_files (self, dir, files, interactive=False):
+ # Delete a list of files
- def __init__ (self, dir, file):
assert os.path.isdir (dir)
- assert os.path.isfile (os.path.join (dir, file))
- self.dir = dir
- self.file = file
+ done = False
+ valid_y = ['Y', 'YES']
+ valid_n = ['N', 'NO', '']
- basename = get_basename (file)
- self.likely_files = find_likely_files (dir, file)
+ if interactive:
+ while not done:
+ print 'Do you want to delete the following?:'
+ for f in files:
+ print f
+ s = raw_input ('Delete [y/N]: ').upper()
- def __list_eq (self, l1, l2):
+ if s in valid_y + valid_n:
+ done = True
- if len(l1) != len(l2):
- return False
+ if s in valid_n:
+ return SUCCESS
- for e in l1:
- if e not in l2:
- return False
+ for f in files:
+ try:
+ os.remove (os.path.join (dir, f))
+ logger.addMessage ('Deleteing: %s' % os.path.join (dir, f), RarslaveLogger.MessageType.Debug)
+ except:
+ logger.addMessage ('Failed to delete: %s' % os.path.join (dir, f),
+ RarslaveLogger.MessageType.Fatal)
+ return -EDELETE
- return True
+ return SUCCESS
- def __eq__ (self, rhs):
- return self.__list_eq (self.likely_files, rhs.likely_files)
+ def runDelete (self):
+ deleteable_files = self.__find_deleteable_files ()
+ ret = self.__delete_list_of_files (self.dir, deleteable_files, options.interactive)
- def run_all (self):
- par2files = find_par2_files (self.likely_files)
- par2head = par2files[0]
+ return ret
- join = is_noextract (self.likely_files)
+ def run_all (self):
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
# Repair Stage
- repairer = RarslaveRepairer (self.dir, par2head, join)
- ret = repairer.checkAndRepair ()
+ ret = self.runCheckAndRepair ()
if ret != SUCCESS:
- logger.addMessage ('Repair stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ logger.addMessage ('Repair stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
return -ECHECK
+ self.__update_name_matches ()
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
# Extraction Stage
- EXTRACT_DIR = options.extract_dir
- extractor = find_extraction_heads (self.dir, self.likely_files)
- ret = extractor.extract (EXTRACT_DIR)
+ extractor = RarslaveExtractor (self.dir, self.all_p2files, \
+ self.name_matched_files, self.prot_matched_files)
+ ret = extractor.runExtract (options.extract_dir)
if ret != SUCCESS:
- logger.addMessage ('Extraction stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ logger.addMessage ('Extraction stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
return -EEXTRACT
+ self.__update_name_matches ()
+ all_files = no_duplicates (self.name_matched_files + self.prot_matched_files)
+
# Deletion Stage
- DELETE_INTERACTIVE = options.interactive
- deleteable_files = find_deleteable_files (self.likely_files)
- ret = delete_list (self.dir, deleteable_files, DELETE_INTERACTIVE)
+ ret = self.runDelete ()
if ret != SUCCESS:
- logger.addMessage ('Deletion stage failed for: %s' % par2head, RarslaveLogger.MessageType.Fatal)
+ logger.addMessage ('Deletion stage failed for: %s' % self.p2file, RarslaveLogger.MessageType.Fatal)
return -EDELETE
- logger.addMessage ('Successfully completed: %s' % par2head)
+ logger.addMessage ('Successfully completed: %s' % self.p2file)
return SUCCESS
-def delete_list (dir, files, interactive=False):
- # Delete a list of files
+def run_command (cmd, indir=None):
+ # Runs the specified command-line in the directory given (or, in the current directory
+ # if none is given). It returns the status code given by the application.
- assert os.path.isdir (dir)
+ pwd = os.getcwd ()
- done = False
- valid_y = ['Y', 'YES']
- valid_n = ['N', 'NO']
+ if indir != None:
+ assert os.path.isdir (indir) # MUST be a directory!
+ os.chdir (indir)
- if interactive:
- while not done:
- print 'Do you want to delete the following?:'
- printlist (files)
- s = raw_input ('Delete [y/N]: ').upper()
+ print 'RUNNING (%s): %s' % (indir, cmd)
+ ret = os.system (cmd)
+ os.chdir (pwd)
+ return ret
- if s in valid_y + valid_n:
- done = True
+def full_abspath (p):
+ return os.path.abspath (os.path.expanduser (p))
- if s in valid_n:
- return SUCCESS
+def find_par2_files (files):
+ """Find all par2 files in the list $files"""
+
+ PAR2_REGEX = config.get_value ('regular expressions', 'par2_regex')
+ regex = re.compile (PAR2_REGEX, re.IGNORECASE)
+ return [f for f in files if regex.match (f)]
- for f in files:
- os.remove (os.path.join (dir, f))
+def find_all_par2_files (dir):
+ """Finds all par2 files in a directory"""
+ # NOTE: does NOT return absolute paths
- return SUCCESS
+ if not os.path.isdir (os.path.abspath (dir)):
+ raise ValueError # bad directory given
+
+ dir = os.path.abspath (dir)
+ files = os.listdir (dir)
+
+ return find_par2_files (files)
+def no_duplicates (li):
+ """Removes all duplicates from a list"""
+ return list(set(li))
def generate_all_parsets (dir):
# Generate all parsets in the given directory.