仅仅使用了一个java文件，运行main方法即可，需要依赖的jar包是com.alibaba.fastjson（版本1.2.28）和jsoup（版本1.10.2）

如果用了pom，那么就是以下两个：

? 1 2 3 4 5 6 7 8 9 10 <dependency> <groupid>com.alibaba</groupid> <artifactid>fastjson</artifactid> <version>1.2.28</version> </dependency> <dependency> <groupid>org.jsoup</groupid> <artifactid>jsoup</artifactid> <version>1.10.2</version> </dependency>

完整的代码如下：

? 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 package com.tuniu.fcm.facade.ipproxy; import com.alibaba.fastjson.jsonobject; import org.jsoup.jsoup; import org.jsoup.nodes.document; import java.util.arraylist; import java.util.hashmap; import java.util.list; import java.util.map; import java.util.regex.matcher; import java.util.regex.pattern; /** * 获取代理ip，需要 * com.alibaba.fastjson.jsonobject以及jsoup */ public class proxycralwerunusedvpn { threadlocal<integer> localwantednumber = new threadlocal<integer>(); threadlocal<list<proxyinfo>> localproxyinfos = new threadlocal<list<proxyinfo>>(); public static void main(string[] args) { proxycralwerunusedvpn proxycrawler = new proxycralwerunusedvpn(); /** * 想要获取的代理ip个数，由需求方自行指定。（如果个数太多，将导致返回变慢） */ proxycrawler.startcrawler(1); } /** * 暴露给外部模块调用的入口 * @param wantednumber 调用方期望获取到的代理ip个数 */ public string startcrawler(int wantednumber) { localwantednumber.set(wantednumber); kuaidailicom("http://www.xicidaili.com/nn/", 15); kuaidailicom("http://www.xicidaili.com/nt/", 15); kuaidailicom("http://www.xicidaili.com/wt/", 15); kuaidailicom("http://www.kuaidaili.com/free/inha/", 15); kuaidailicom("http://www.kuaidaili.com/free/intr/", 15); kuaidailicom("http://www.kuaidaili.com/free/outtr/", 15); /** * 构造返回数据 */ proxyresponse response = new proxyresponse(); response.setsuccess("true"); map<string, object> datainfomap = new hashmap<string, object>(); datainfomap.put("numfound", localproxyinfos.get().size()); datainfomap.put("pagenum", 1); datainfomap.put("proxy", localproxyinfos.get()); response.setdata(datainfomap); string responsestring = jsonobject.tojson(response).tostring(); system.out.println(responsestring); return responsestring; } private void kuaidailicom(string baseurl, int totalpage) { string ipreg = "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3} \\d{1,6}"; pattern ipptn = pattern.compile(ipreg); for (int i = 1; i < totalpage; i++) { if (getcurrentproxynumber() >= localwantednumber.get()) { return; } try { document doc = jsoup.connect(baseurl + i + "/") .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") .header("accept-encoding", "gzip, deflate, sdch") .header("accept-language", "zh-cn,zh;q=0.8,en;q=0.6") .header("cache-control", "max-age=0") .header("user-agent", "mozilla/5.0 (macintosh; intel mac os x 10_11_4) applewebkit/537.36 (khtml, like gecko) chrome/51.0.2704.103 safari/537.36") .header("cookie", "hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1462812244; _gat=1; _ga=ga1.2.1061361785.1462812244") .header("host", "www.kuaidaili.com") .header("referer", "http://www.kuaidaili.com/free/outha/") .timeout(30 * 1000) .get(); matcher m = ipptn.matcher(doc.text()); while (m.find()) { if (getcurrentproxynumber() >= localwantednumber.get()) { break; } string[] strs = m.group().split(" "); if (checkproxy(strs[0], integer.parseint(strs[1]))) { system.out.println("获取到可用代理ip\t" + strs[0] + "\t" + strs[1]); addproxy(strs[0], strs[1], "http"); } } } catch (exception e) { e.printstacktrace(); } } } private static boolean checkproxy(string ip, integer port) { try { //http://1212.ip138.com/ic.asp 可以换成任何比较快的网页 jsoup.connect("http://1212.ip138.com/ic.asp") .timeout(2 * 1000) .proxy(ip, port) .get(); return true; } catch (exception e) { return false; } } private int getcurrentproxynumber() { list<proxyinfo> proxyinfos = localproxyinfos.get(); if (proxyinfos == null) { proxyinfos = new arraylist<proxyinfo>(); localproxyinfos.set(proxyinfos); return 0; } else { return proxyinfos.size(); } } private void addproxy(string ip, string port, string protocol){ list<proxyinfo> proxyinfos = localproxyinfos.get(); if (proxyinfos == null) { proxyinfos = new arraylist<proxyinfo>(); proxyinfos.add(new proxyinfo(ip, port, protocol)); } else { proxyinfos.add(new proxyinfo(ip, port, protocol)); } } } class proxyinfo { private string username = ""; private string ip; private string password = ""; private string type; private string port; private int is_internet = 1; public proxyinfo(string ip, string port, string type) { this.ip = ip; this.type = type; this.port = port; } public string getusername() { return username; } public void setusername(string username) { this.username = username; } public string getip() { return ip; } public void setip(string ip) { this.ip = ip; } public string getpassword() { return password; } public void setpassword(string password) { this.password = password; } public string gettype() { return type; } public void settype(string type) { this.type = type; } public string getport() { return port; } public void setport(string port) { this.port = port; } public int getis_internet() { return is_internet; } public void setis_internet(int is_internet) { this.is_internet = is_internet; } } class proxyresponse { private string success; private map<string, object> data; public string getsuccess() { return success; } public void setsuccess(string success) { this.success = success; } public map<string, object> getdata() { return data; } public void setdata(map<string, object> data) { this.data = data; } }

以上这篇java代理实现爬取代理ip的示例就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持服务器之家。

原文链接：https://blog.csdn.net/sdfiiiiii/article/details/70432060