Subversion Repositories programming

Rev

Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
129 ira 1
#!/usr/bin/env python
2
 
3
# Copyright: Ira W. Snyder (devel@irasnyder.com)
4
# Start Date: 2005-10-13
5
# End Date:
6
# License: GNU General Public License v2 (or at your option, any later version)
7
#
8
# Changelog Follows:
9
# - 2005-10-13
10
# - Added get_par2_filenames() to parse par2 files
11
# - Added the parset object to represent each parset.
12
#
13
# - 2005-10-14
14
# - Finished the parset object. It will now verify and extract parsets.
15
# - Small changes to the parset object. This makes the parjoin part
16
#   much more reliable.
17
# - Added the OptionParser to make this nice to run at the command line.
18
# - Made recursiveness an option.
19
# - Made start directory an option.
20
# - Check for appropriate programs before starting.
21
#
22
 
23
################################################################################
24
# REQUIREMENTS:
25
#
26
# This code requires the programs cfv, par2repair, lxsplit, and rar to be able
27
# to function properly. I will attempt to check that these are in your path.
28
################################################################################
29
 
30
################################################################################
31
# Global Variables
32
################################################################################
33
WORK_DIR = '~/downloads/usenet'
34
################################################################################
35
 
36
################################################################################
37
# The PAR2 Parser
38
#
39
# This was stolen from cfv (see http://cfv.sourceforge.net/ for a copy)
40
################################################################################
41
 
42
import struct, errno
43
 
44
# We always want to do crc checks
45
docrcchecks = True
46
 
47
def chompnulls(line):
48
    p = line.find('\0')
49
    if p < 0: return line
50
    else:     return line[:p]
51
 
52
def get_par2_filenames(filename):
53
    """Get all of the filenames that are protected by the par2
54
    file given as the filename"""
55
 
56
    try:
57
        file = open(filename, 'rb')
58
    except:
59
        print 'Could not open %s' % (filename, )
60
        return []
61
 
62
    pkt_header_fmt = '< 8s Q 16s 16s 16s'
63
    pkt_header_size = struct.calcsize(pkt_header_fmt)
64
    file_pkt_fmt = '< 16s 16s 16s Q'
65
    file_pkt_size = struct.calcsize(file_pkt_fmt)
66
    main_pkt_fmt = '< Q I'
67
    main_pkt_size = struct.calcsize(main_pkt_fmt)
68
 
69
    seen_file_ids = {}
70
    expected_file_ids = None
71
    filenames = []
72
 
73
    while 1:
74
        d = file.read(pkt_header_size)
75
        if not d:
76
            break
77
 
78
        magic, pkt_len, pkt_md5, set_id, pkt_type = struct.unpack(pkt_header_fmt, d)
79
 
80
        if docrcchecks:
81
            import md5
82
            control_md5 = md5.new()
83
            control_md5.update(d[0x20:])
84
            d = file.read(pkt_len - pkt_header_size)
85
            control_md5.update(d)
86
 
87
            if control_md5.digest() != pkt_md5:
88
                raise EnvironmentError, (errno.EINVAL, \
89
                    "corrupt par2 file - bad packet hash")
90
 
91
        if pkt_type == 'PAR 2.0\0FileDesc':
92
            if not docrcchecks:
93
                d = file.read(pkt_len - pkt_header_size)
94
 
95
            file_id, file_md5, file_md5_16k, file_size = \
96
                struct.unpack(file_pkt_fmt, d[:file_pkt_size])
97
 
98
            if seen_file_ids.get(file_id) is None:
99
                seen_file_ids[file_id] = 1
100
                filename = chompnulls(d[file_pkt_size:])
101
                filenames.append(filename)
102
 
103
        elif pkt_type == "PAR 2.0\0Main\0\0\0\0":
104
            if not docrcchecks:
105
                d = file.read(pkt_len - pkt_header_size)
106
 
107
            if expected_file_ids is None:
108
                expected_file_ids = []
109
                slice_size, num_files = struct.unpack(main_pkt_fmt, d[:main_pkt_size])
110
                num_nonrecovery = (len(d)-main_pkt_size)/16 - num_files
111
 
112
                for i in range(main_pkt_size,main_pkt_size+(num_files+num_nonrecovery)*16,16):
113
                    expected_file_ids.append(d[i:i+16])
114
 
115
        else:
116
            if not docrcchecks:
117
                file.seek(pkt_len - pkt_header_size, 1)
118
 
119
    if expected_file_ids is None:
120
        raise EnvironmentError, (errno.EINVAL, \
121
            "corrupt or unsupported par2 file - no main packet found")
122
 
123
    for id in expected_file_ids:
124
        if not seen_file_ids.has_key(id):
125
            raise EnvironmentError, (errno.EINVAL, \
126
                "corrupt or unsupported par2 file - " \
127
                "expected file description packet not found")
128
 
129
    return filenames
130
 
131
################################################################################
132
# The parset object
133
#
134
# This is an object based representation of a parset, and will verify itself
135
# and extract itself, if possible.
136
################################################################################
137
 
138
import os, glob
139
 
140
class parset:
141
    def __init__(self, par_filename):
142
        self.parfile = par_filename
143
        self.extra_pars = []
144
        self.files = False
145
        self.used_parjoin = False
146
        self.verified = False
147
        self.extracted = False
148
 
149
    def get_filenames(self):
150
        return get_par2_filenames(parfile)
151
 
152
    def all_there(self):
153
        """Check if all the files for the parset are present.
154
        This will help us decide which par2 checker to use first"""
155
        for f in self.files:
156
            if not os.path.isfile(f):
157
                return False
158
 
159
        # The files were all there
160
        return True
161
 
162
    def verify(self):
163
        """This will verify the parset by the most efficient method first,
164
        and then move to a slower method if that one fails"""
165
 
166
        retval = False #not verified yet
167
 
168
        # if all the files are there, try verifying fast
169
        if self.all_there():
170
            retval = self.__fast_verify()
171
 
172
            if retval == False:
173
                # Failed to verify fast, so try it slow, maybe it needs repair
174
                retval = self.__slow_verify()
175
 
176
        # If we've got a video file, maybe we should try to parjoin it
177
        elif self.__has_video_file():
178
            retval = self.__parjoin()
179
 
180
        else: #not all there, maybe we can slow-repair
181
            retval = self.__slow_verify()
182
 
183
        self.verified = retval
184
        return self.verified
185
 
186
    def __fast_verify(self):
187
        retval = os.system('cfv -v -f "%s"' % (self.parfile, ))
188
 
189
        if retval == 0:
190
            return True #success
191
 
192
        return False #failure
193
 
194
    def __slow_verify(self):
195
        retval = os.system('par2repair "%s"' % (self.parfile, ))
196
 
197
        if retval == 0:
198
            return True #success
199
 
200
        return False #failure
201
 
202
    def __parjoin(self):
203
        retval = os.system('lxsplit -j "%s.001"' % (self.files[0], ))
204
 
205
        retval = self.__fast_verify()
206
 
207
        if retval == False:
208
            # Failed to verify fast, so try it slow, maybe it needs repair
209
            retval = self.__slow_verify()
210
 
211
        if retval == False: # failed to verify, so remove the lxsplit created file
212
            os.remove(self.files[0])
213
 
214
        self.used_parjoin = retval
215
        self.verified = retval
216
        return self.verified
217
 
218
    def __has_video_file(self):
219
        for f in self.files:
220
            if os.path.splitext(f)[1] in ('.avi', '.ogm', '.mkv'):
221
                return True
222
 
223
        return False
224
 
225
    def __remove_currentset(self):
226
        """Remove all of the files that are extractable, as well as the pars.
227
        Leave everything else alone"""
228
 
229
        if not self.extracted:
230
            print 'Did not extract yet, not removing currentset'
231
            return
232
 
233
        # remove the main par
234
        os.remove(self.parfile)
235
 
236
        # remove all of the extra pars
237
        for i in self.extra_pars:
238
            os.remove(i)
239
 
240
        # remove any rars that are associated (leave EVERYTHING else)
241
        for i in self.files:
242
            if i[-3:] == 'rar':
243
                os.remove(i)
244
 
245
        # remove any .0?? files (from parjoin)
246
        if self.used_parjoin:
247
            for i in os.listdir(os.getcwd()):
248
                if i != self.files[0] and self.files[0] in i:
249
                    os.remove(i)
250
 
251
        # remove any temp repair files
252
        for i in glob.glob('*.1'):
253
            os.remove(i)
254
 
255
    def __get_extract_file(self):
256
        """Find the first extractable file"""
257
        for i in self.files:
258
            if os.path.splitext(i)[1] == '.rar':
259
                return i
260
 
261
        return None
262
 
263
    def extract(self):
264
        """Attempt to extract all of the files related to this parset"""
265
        if not self.verified:
266
            self.extracted = False
267
            print 'Not (successfully) verified, not extracting'
268
            return False #failed to extract
269
 
270
        extract_file = self.__get_extract_file()
271
 
272
        if extract_file != None:
273
            retval = os.system('rar e -o+ "%s"' % (extract_file, ))
274
 
275
            if retval != 0:
276
                print 'Failed to extract'
277
                self.extracted = False
278
                return self.extracted
279
 
280
        # we extracted ok, so remove the currentset
281
        self.extracted = True
282
        self.__remove_currentset()
283
 
284
        return self.extracted
285
 
286
 
287
################################################################################
288
# The rarslave program itself
289
################################################################################
290
 
291
import os, sys, glob
292
from optparse import OptionParser
293
 
294
def check_required_progs():
295
    """Check if the required programs are installed"""
296
 
297
    shell_not_found = 32512
298
    needed = []
299
 
300
    if os.system('cfv --help > /dev/null 2>&1') == shell_not_found:
301
        needed.append('cfv')
302
 
303
    if os.system('par2repair --help > /dev/null 2>&1') == shell_not_found:
304
        needed.append('par2repair')
305
 
306
    if os.system('lxsplit --help > /dev/null 2>&1') == shell_not_found:
307
        needed.append('lxpsplit')
308
 
309
    if os.system('rar --help > /dev/null 2>&1') == shell_not_found:
310
        needed.append('rar')
311
 
312
    if needed:
313
        for n in needed:
314
            print 'Needed program "%s" not found in $PATH' % (n, )
315
 
316
        sys.exit(1)
317
 
318
def get_parsets():
319
    """Get a representation of each parset in the current directory, and
320
    return them as a list of parset instances"""
321
 
322
    par2files =  glob.glob('*.par2')
323
    par2files += glob.glob('*.PAR2')
324
 
325
    parsets = []
326
 
327
    for i in par2files:
328
        filenames = get_par2_filenames(i)
329
        create_new = True
330
 
331
        # if we already have an instance for this set, append
332
        # this par file to the extra_pars field
333
        for j in parsets:
334
            if j.files == filenames:
335
                j.extra_pars.append(i)
336
                create_new = False
337
 
338
        # we haven't seen this set yet, so we'll create it now
339
        if create_new == True:
340
            cur = parset(i)
341
            cur.files = filenames
342
            parsets.append(cur)
343
 
344
    return parsets
345
 
346
def directory_worker(dir):
347
    """Attempts to find, verify, and extract every parset in the directory
348
    given as a parameter"""
349
 
350
    cwd = os.getcwd()
351
    os.chdir(dir)
352
 
353
    parsets = get_parsets()
354
 
355
    # Verify each parset
356
    for p in parsets:
357
        p.verify()
358
 
359
    # Attempt to extract each parset
360
    for p in parsets:
361
        p.extract()
362
 
363
    os.chdir(cwd)
364
 
365
def main():
366
 
367
    # Build the OptionParser
368
    parser = OptionParser()
369
    parser.add_option('-n', '--not-recursive', action='store_false', dest='recursive',
370
                      default=True, help="don't run recursively")
371
    parser.add_option('-d', '--start-dir', dest='work_dir', default=WORK_DIR,
372
                      help='start running at DIR', metavar='DIR')
373
 
374
    # Parse the given options
375
    (options, args) = parser.parse_args()
376
 
377
    # Fix up the working directory
378
    options.work_dir = os.path.abspath(os.path.expanduser(options.work_dir))
379
 
380
    # Check that we have the required programs installed
381
    check_required_progs()
382
 
383
    # Run rarslave!
384
    if options.recursive:
385
        for root, dirs, files in os.walk(options.work_dir):
386
            directory_worker(root)
387
    else:
388
        directory_worker(options.work_dir)
389
 
390
if __name__ == '__main__':
391
    main()
392