python爬虫-梨视频短视频爬取(线程池)

示例代码

import requestsfrom lxml import etreeimport randomfrom multiprocessing.dummy import Pool# 多进程要传的方法，多进程pool.map()传的第二个参数是一个迭代器对象# 而传的get_video方法也要有一个迭代器参数def get_video(dic): headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56' } video_data = requests.get(url = dic['url'] , headers = headers).content print(dic['name']+'开始下载') # 有的文件名中包含空格，在并发执行时会报错，这里用随机数给文件起名了 #path = dic['name']+'.mp4'会报错 path = "./lishipin/"+str(int(random.random()*100)) + '.mp4' with open(path,'wb') as fp: fp.write(video_data) print(dic['name']+'下载成功')def main(): # web_url:梨视频官网 web_url = 'https:///' urllist.append(prefix+i) return urllist# 获取视频链接并下载（写入到硬盘）def getvideo(url): res=requests.get(url,headers) mp4url=re.findall('srcUrl="(.*?\.mp4)"',res.text,re.S)[0] video=requests.get(mp4url) m = hashlib.md5() m.update(url.encode('utf-8')) m.update(str(time.time()).encode('utf-8')) filename = r'%s.mp4' % m.hexdigest() print(filename) with open("/home/tony/文档/爬虫视频/%s.mp4"%filename,'wb') as f: f.write(video.content)def main(): video_urllist=geturls(mainurl) for i in video_urllist: getvideo(i)if __name__=='__main__': main()

到此这篇关于python爬虫线程池案例详解(梨视频短视频爬取)的文章就介绍到这了,更多相关python爬虫梨视频短视频爬取内容请搜索以前的文章或继续浏览下面的相关文章希望大家以后多多支持！