简单的crawler,python
以下是十分简单的python写的爬虫小脚本。import os,reimport urlparse,urllib,urllib2import hashlib,Queueclass Request(object): def __init__(self): self.url= '' self.error_url = [] self.undownload_url = [] self.downloaded_url = [] self.headers={'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6)\ Gecko/20091201 Firefox/3.5.6', 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' } def read_url_from_file(self): urls = [] try: r = open('urls.txt','r') urls = r.readlines() r.close() except IOError: url = raw_input("no file urls.txt,please input a url >>> ") if not url.lower().startswith('http://'): url = 'http://'+url urls.append(url) return urls def is_download_url(self): return self.url in self.downloaded_url def get(self): ok = False req = urllib2.Request(self.url,None,self.headers) for i in range(3): temp = urllib2.urlopen(req) if temp.getcode()==200: if 'content-type' in temp.headers.keys(): content_type = temp.headers.get('content-type') if 'text' in content_type: return temp.read() else: return temp.read() return '' def find_links(self): com = re.compile(r"""<a[^>]*?href\s*=\s*['"]?([^'"\s>]{1,500})['">\s]""",re.I|re.M|re.S) links = com.findall(self.content) links = set(links) links = list(links) full_links = for i in full_links: if i not in self.undownload_url and i not in self.downloaded_url: self.undownload_url.append(i) def save_page(self,page_source,filename): dirname = filename.split('/') if not os.path.isdir(dirname): os.mkdir(dirname) w = open(filename,'w') w.write(page_source) w.close() def save_result(self,urls,filename): w = open(filename,'w') w.write('\n'.join(urls)) w.close() def run(self): self.page = 0 self.ldir = 0 self.num_per_dir = 100 self.undownload_url = self.read_url_from_file() self.url_to_path = {} try: while self.undownload_url: self.url = self.undownload_url.pop(0) self.url = self.url.replace('\n','') order = self.page+self.ldir*self.num_per_dir print (" %4d download... %s " % (order,self.url) ) self.content = self.get() self.downloaded_url.append(self.url) if self.content: self.find_links() filename = str(self.ldir)+'/'+str(self.page) self.save_page(self.content,filename) self.url_to_path = self.url self.page += 1 if self.page >= self.num_per_dir: self.page -= self.num_per_dir self.ldir += 1 else: self.error_url.append(self.url) print ("finished !") except: print ("save downloaded urls into downloaded file") self.save_result(self.downloaded_url,'downloaded') print ("save error urls into error file") self.save_result(self.undownload_url,'undownload') print ("save url to file dictonary into url_to_file") self.save_result( for x in self.url_to_path.keys()],'url_to_file') print ("total try to download %d urls" % order) aa = Request()aa.run()
页:
[1]