本文实例讲述了Python爬取国外天气预报网站的方法。分享给大家供大家参考。具体如下：

crawl_weather.py如下：

#encoding=utf-8import httplibimport urllib2import timefrom threading import Threadimport threadingfrom Queue import Queuefrom time import sleepimport reimport copylang = "fr"count = 0class Location: # Location(False, "中国", "北京", "zh") # Location(True, "", "亚洲", "zh") def __init__(self, is_beyond_country, country_name, loc_name, lang): self.country_name = country_name self.loc_name = loc_name self.lang = lang self.is_beyond_country = is_beyond_countryprn_lock = threading.RLock()def GetLocationURLs(url, recursive): global count if url.find("weather-forecast") != -1: count = count + 1 if count % 500 == 0: prn_lock.acquire() print "count:%d" % (count) prn_lock.release() return [url] page = urllib2.urlopen(url).read() time.sleep(0.01) #"<h6><a href=\"http:///zh/cn/andingmen/57494/weather-forecast/57494"page = urllib2.urlopen(root_url).read()print page'''

FetchLocation.py如下：

#encoding=utf-8import sysimport httplibimport urllib2import timefrom threading import Threadimport threadingfrom Queue import Queuefrom time import sleepimport reimport copyfrom xml.dom import minidomimport HTMLParserimport datetimeq = Queue()locks = [threading.RLock() for i in range(2)]ThreadNumber = 20locations = {}conds = {}def FindCountryBreadCrumbs(page): lines = page.splitlines() count = 0 start = -1 opened = False for line in lines: if line.find("<ul id=\"country-breadcrumbs\">") != -1: start = count opened = True if opened and line.find("</ul>") != -1: end = count opened = False count = count + 1 return "\n".join(lines[start: (end + 1)])def GetText(nodelist): rc = [] for node in nodelist: if node.nodeType == node.TEXT_NODE: rc.append(HTMLParser.HTMLParser().unescape(node.data)) return ''.join(rc)def FindCondition(page): pat = "<span class=\"cond\">(.*?)</span>" cds = re.findall(pat, page) cds = [HTMLParser.HTMLParser().unescape(cd).encode("utf-8") for cd in cds] return cds def ExtractInfo(url): try: page = urllib2.urlopen(url).read() except Exception, e: return [] text = FindCountryBreadCrumbs(page) text = HTMLParser.HTMLParser().unescape(text) dom = minidom.parseString(text.encode("utf-8")) locs = [] lis = dom.getElementsByTagName("li") for li in lis: adr_list = li.getElementsByTagName("a") if adr_list: locs.append(GetText(adr_list[0].childNodes).encode("utf-8")) strs = li.getElementsByTagName("strong") if strs: locs.append(GetText(strs[0].childNodes).encode("utf-8")) cds = FindCondition(page) return locs, cdsdef AddMap(lst, m): for x in lst: if m.get(x) == None: m[x] = 1def working(): while True: urls = q.get() #print len(urls) m = {} m2 = {} count = 0 for url in urls: count = count + 1 #print "%d/%d" % (count, len(urls)) locs, cds = ExtractInfo(url) AddMap(locs, m) AddMap(cds, m2) locks[1].acquire() AddMap(m.keys(), locations) AddMap(m2.keys(), conds) locks[1].release() q.task_done()def main(): if len(sys.argv) < 2: exit() loc_path = sys.argv[1] fp = open(loc_path, "r") urls = [line.strip() for line in fp] fp.close() #urls = urls[0:1000] blocks = len(urls) / ThreadNumber + 1 for start in range(0, len(urls), blocks): end = start + blocks if end > len(urls): end = len(urls) q.put(urls[start:end]) for i in range(ThreadNumber): t = Thread(target=working) t.setDaemon(True) t.start() q.join() fp = open("location_name.fr", "w") fp.write("\n".join(locations.keys())) fp.close() fp = open("conditions.fr", "w") fp.write("\n".join(conds.keys())) fp.close()if __name__ == '__main__': main()

希望本文所述对大家的python程序设计有所帮助。