Python抓取图片
# -*- coding: cp936 -*-importurllibimport urllib2import randomimport os,sysfrom sgmllib import SGMLParserclass URLLister(SGMLParser): '''获取html中的图片地址\url地址，装入list中''' def reset(self): SGMLParser.reset(self) self.img = [] self.urls = [] def start_img(self, attrs): img = if img: self.img.extend(img) def start_a(self, attrs): href = if href: self.urls.extend(href)def get_docum(url): #url=url+'//' sock=urllib.urlopen(url) file=sock.read() sock.close() return filedef is_img(url): global imglenth request=urllib2.Request(url) opener=urllib2.build_opener() try: con=opener.open(request) Type=con.headers.dict['content-type'][:5] #判断链接返回的 content-type是不是图片。 Length =int(con.headers.dict['content-length'])#判断图片大小 if Length>imglenth: return Type else: return 0 except: print sys.exc_info(),sys.exc_info()##一般来说这样就足够了 print '该图片无法在服务器找到或者图片地址无法识别！' print url def get_file_name(ospath,imgname,num): #name = 'P'+str(random.randint(10000000,99999999)) #filepath = "%s%s.%s" % (ospath,name,(imgname.split('.'))[-1]) #保留原文件名 idx=imgname.rfind("/") filename=imgname if idx > -1: filename=imgname filepath=ospath+'%d_'%num+filename print filepath return filepath def get_img(rq): parser = URLLister(); doc=get_docum(rq); parser.feed(doc); img = parser.img parser.close() for i in range(0,len(img)): if img!='http':#处理绝对路径 img=rq+img return imgdef get_url(rq): parser = URLLister(); doc=get_docum(rq); parser.feed(doc); urls = parser.urls parser.close() for i in range(0,len(urls)): if urls != 'http': #处理绝对路径 urls = rq+urls return urlsdef depth(url,dep,ospath): '''三个参数分别是 url ： 需要下载的网站地址 dep ：需要遍历的深度 ospath：图片下载的本地文件夹 ''' global num if dep<=0: return 0 else: img=get_img(url) for j in range(0,len(img)): if is_img(img) == 'image': filepath = get_file_name(ospath,img,num+1); if (os.path.exists(filepath)): pass try: urllib.urlretrieve(img, filepath) print '已经下载好第%d张图片'%(num+1) num+=1 except: print '该图片无法下载或者图片地址无法识别！' print img else: pass urls=get_url(url) if len(urls)>0: for url in urls: depth(url,dep-1,ospath) else: return 0 return 1if __name__ == '__main__': imglenth = 1 #设置需要下载的图片大小。 num=0 depth('http://tieba.baidu.com/f?kz=820731958',1,"E:\\python\\GetImageFromWeb\\img\\") print '********************************我爬完了！！******************************************'
页:
[1]