新闻抓取爬虫 PPT
不用多言，请见附件 ：）上slideshare view-> http://www.slideshare.net/edisonlz/ppt-4395676
补充，不使用分词的，很简单，但效果一般的排重算法如下：
#encoding = utf8import re import sys,osimport rechars = []def generate_chars(strList): """ 创建字典 """ for sr in strList: for char in sr: if char not in chars: chars.append(char)def vector(sr): """ 将字符串转换为向量空间 """ vec = [] for char in chars: if sr.find(char) !=-1: vec.append(1) else: vec.append(0) return vecdef intersaction(str1,str2): """ 计算交集 """ v1 =vector(str1) v2= vector(str2) rv = 0 for i in xrange(len(v1)): rv += v1 * v2 return rvdef detect_repeat(strList): """ 检测字符串字符匹配 """ chars = generate_chars(strList) #临界值 critical = 3 results = [] for st in strList: if all(intersaction(st,rt) < criticalfor rt in results): results.append(st) #返回检测结果 return resultsif __name__ =="__main__": strLists = results = detect_repeat(strLists) print "result " + " > " * 20 for ru in results: print ru.encode("utf-8")
页:
[1]