多线程下载文件
#!/usr/bin/python#coding=gbkimport reimport osimport sysimport timeimport globimport stringimport socketimport getoptimport urllibimport urllib2import threadingfrom sgmllib import SGMLParser############################################################################### self-defined exception classes##############################################################################class ConnectionError(Exception): passclass URLUnreachable(Exception):passclass CanotDownload(Exception):pass############################################################################### multiple threads download module starts here##############################################################################class HttpGetThread(threading.Thread): def __init__(self, name, url, filename, range=0): #print " name ",name #print " url ",url #print " filename ",filename #print " range ",range threading.Thread.__init__(self, name=name) self.url = url self.filename = filename self.range = range self.totalLength = range - range +1 try: self.downloaded = os.path.getsize(self.filename) except OSError: self.downloaded = 0 self.percent = self.downloaded/float(self.totalLength)*100 self.headerrange = (self.range+self.downloaded, self.range) self.bufferSize = 8192 def run(self): try: self.downloaded = os.path.getsize(self.filename) except OSError: self.downloaded = 0 self.percent = self.downloaded/float(self.totalLength)*100 #self.headerrange = (self.range+self.downloaded, self.range) self.bufferSize = 8192 #request = urllib2.Request(self.url) #request.add_header('Range', 'bytes=%d-%d' %self.headerrange) downloadAll = False retries = 1 while not downloadAll: if retries > 10: break try: self.headerrange = (self.range+self.downloaded, self.range) request = urllib2.Request(self.url) request.add_header('Range', 'bytes=%d-%d' %self.headerrange) conn = urllib2.urlopen(request) startTime = time.time() data = conn.read(self.bufferSize) while data: f = open(self.filename, 'ab') f.write(data) f.close() self.time = int(time.time() - startTime) self.downloaded += len(data) self.percent = self.downloaded/float(self.totalLength) *100 data = conn.read(self.bufferSize) downloadAll = True except Exception, err: retries += 1 time.sleep(1) continue#分割文件方便多线程下载def Split(size,blocks): ranges = [] blocksize = size / blocks for i in xrange(blocks-1): ranges.append((i*blocksize,blocksize*i+blocksize-1)) ranges.append(( blocksize*(blocks-1), size-1)) #print ranges return ranges#获得文件大小def GetHttpFileSize(url): length = 0 try: conn = urllib.urlopen(url) headers = conn.info().headers for header in headers: if header.find('Length') != -1: length = header.split(':')[-1].strip() length = int(length) except Exception, err: pass return lengthdef hasLive(ts): for t in ts: print "\n thread name ",t.getName()," alive ",t.isAlive() if t.isAlive(): return True return False#def MyHttpGet(url, output=None, connections=4): """ arguments: url, in GBK encoding output, default encoding, do no convertion connections, integer """ length = GetHttpFileSize(url) startTime = time.time() #开始时间 #print " startTime ",startTime mb = length/1024/1024.0 if length == 0: raise URLUnreachable blocks = connections if output: filename = output else: output = url.split('/')[-1] ranges = Split(length, blocks) names = [] #names = ["%s_%d" %(filename,i) for i in xrange(blocks)] for i in xrange(blocks): names.append("%s_%d" %(filename,i)) ts = [] #print "+++++++++++++++++++++++++++ blocks ",blocks for i in xrange(blocks): t = HttpGetThread(" 下载线程 "+str(i), url, names, ranges) t.setDaemon(True) t.start() ts.append(t) live = hasLive(ts) startSize = sum() # 已下载多少 #print "++++++++++ startSize ",startSize etime = 0 # rate = 0 # 下载速度 * while live: try: etime = time.time() - startTime d = sum()/float(length)*100 downloadedThistime = sum()-startSize try: rate = downloadedThistime / float(etime)/1024 except: rate = 0.0 progressStr = u'\rFilesize: %d(%.2fM)Downloaded: %.2f%%Avg rate: %.1fKB/s' %(length, mb, d, rate) sys.stdout.write(progressStr) sys.stdout.flush() #sys.stdout.write('\b'*(len(progressStr)+1)) live = hasLive(ts) time.sleep(0.8) except KeyboardInterrupt: print print "Exit..." for n in names: try: os.remove(n) except: pass sys.exit(1) print etime = time.time() - startTime print " endTime ",time.time() printu'耗时： %d:%d, 平均速度：%.2fKB/s' %(int(etime)/60, int(etime)%60,rate) f = open(filename, 'wb') for n in names: f.write(open(n,'rb').read()) try: os.remove(n) except: pass f.close()if __name__ == "__main__": url = "http://www.jszg.com.cn/webpage/网页作业/天黑黑.mp3" output = "孙燕姿---天黑黑.mp3" connections = 10 MyHttpGet(url,output,connections);
页:
[1]