C#插件火车头采集器动态切换代理IP，及自动切换UserAgent 通过代理供应商接口，实时获取代理IP，每个IP使用1分钟，1分钟后重新获取新IP。整合网上抄来的随机UserAgent，合成了一个火车头插件。using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using SpiderInterface; using System.Net; using System.IO; namespace PluginSample { public class Plugin1 : IHTTPTamper { string[] agentList =new string[] { "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60", "Opera/8.0 (Windows NT 5.1; U; en)", "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0", "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36" }; public void CheckCacheDir() { string Dir = "D:\\IPCACHE\\"; if (!Directory.Exists(Dir)) Directory.CreateDirectory(Dir); } public string GetUrlContent(string url) { using(WebClient wc = new WebClient()){ try { return Encoding.GetEncoding("UTF-8").GetString(wc.DownloadData(string.Format(url))); } catch (Exception e) { throw new Exception(e.ToString()); } } } public string ReadTxt() { string Result = ""; try { string filename = "D:\\IPCACHE\\" + DateTime.Now.ToString("yyyyMMddHHmm") + ".txt"; using (StreamReader sr = new StreamReader(filename)) { Result = sr.ReadToEnd(); } } catch { Result = "缓存不存在"; } return Result; } public string WriteTxt(string val) { try { string filename = "D:\\IPCACHE\\" + DateTime.Now.ToString("yyyyMMddHHmm") + ".txt"; using (StreamWriter sr = new StreamWriter(filename)) { sr.Write(val); } } catch { return "fail"; } return "Success"; } public string GetProxyIp(){ string ProxyApi = "代理IP获取的API, 结果应该是 0.0.0.0.0:xxxx"; CheckCacheDir(); string ip = ""; //检查缓存 string ProxyCache = ReadTxt(); if (ProxyCache != "缓存不存在" && ProxyCache != "") { ip = ProxyCache; } else { ip = GetUrlContent(ProxyApi); WriteTxt(ip); } return ip; } /// <summary> /// 处理下载前的request /// </summary> /// <param name="response"></param> public void BeforeRequest(RequestEntry request) { //Console.WriteLine("BeforeRequest："+request.Url); //request.Referer=""; //request.Headers. Random r = new Random(); string Agent = agentList[r.Next(agentList.Length)]; request.Headers["User-Agent"] =Agent; var proxy = new WebProxy(GetProxyIp(), true); request.WebProxy=proxy; } /// <summary> /// 处理下载完成后的http响应,网址、默认页、多页、内容分页 /// </summary> /// <param name="response"></param> public void AfterResponse(ResponseEntry response) { Console.WriteLine("AfterResponse：" + response.Url); } } }