本文实例讲述了Python实现的爬虫功能。分享给大家供大家参考，具体如下：

主要用到urllib2、BeautifulSoup模块

#encoding=utf-8import reimport requestsimport urllib2import datetimeimport MySQLdbfrom bs4 import BeautifulSoupimport sysreload(sys)sys.setdefaultencoding("utf-8")class Splider(object): def __init__(self): print u'开始爬取内容...' ##用来获取网页源代码 def getsource(self,url): headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2652.0 Safari/537.36'} req = urllib2.Request(url=url,headers=headers) socket = urllib2.urlopen(req) content = socket.read() socket.close() return content ##changepage用来生产不同页数的链接 def changepage(self,url,total_page): now_page = int(re.search('page/(\d+)',url,re.S).group(1)) page_group = [] for i in range(now_page,total_page+1): link = re.sub('page/(\d+)','page/%d' % i,url,re.S) page_group.append(link) return page_group #获取字内容 def getchildrencon(self,child_url): conobj = {} content = self.getsource(child_url) soup = BeautifulSoup(content, 'html.parser', from_encoding='utf-8') content = soup.find('div',{'class':'c-article_content'}) img = re.findall('src="(.*?)"',str(content),re.S) conobj['con'] = content.get_text() conobj['img'] = (';').join(img) return conobj ##获取内容 def getcontent(self,html_doc): soup = BeautifulSoup(html_doc, 'html.parser', from_encoding='utf-8') tag = soup.find_all('div',{'class':'promo-feed-headline'}) info = {} i = 0 for link in tag: info[i] = {} title_desc = link.find('h3') info[i]['title'] = title_desc.get_text() post_date = link.find('div',{'class':'post-date'}) pos_d = post_date['data-date'][0:10] info[i]['content_time'] = pos_d info[i]['source'] = 'whowhatwear' source_link = link.find('a',href=re.compile(r"section=fashion-trends")) source_url = 'http:///section/fashion-trends/page/1' jikesplider = Splider() all_links = jikesplider.changepage(url,p_num) for link in all_links: print u'正在处理页面：' + link html = jikesplider.getsource(link) info = jikesplider.getcontent(html) classinfo.append(info) jikesplider.saveinfo(classinfo)

更多关于Python相关内容可查看本站专题：《Python Socket编程技巧总结》、《Python数据结构与算法教程》、《Python函数使用技巧总结》、《Python字符串操作技巧汇总》、《Python入门与进阶经典教程》及《Python文件与目录操作技巧汇总》

希望本文所述对大家Python程序设计有所帮助。