通过代理供应商接口，实时获取代理IP，每个IP使用1分钟，1分钟后重新获取新IP。
整合网上抄来的随机UserAgent，合成了一个火车头插件。
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using SpiderInterface;
using System.Net;
using System.IO;
namespace PluginSample
{
public class Plugin1 : IHTTPTamper
{
string[] agentList =new string[] {
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
"Opera/8.0 (Windows NT 5.1; U; en)",
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36"
};
public void CheckCacheDir() {
string Dir = "D:\\IPCACHE\\";
if (!Directory.Exists(Dir)) Directory.CreateDirectory(Dir);
}
public string GetUrlContent(string url)
{
using(WebClient wc = new WebClient()){
try
{
return Encoding.GetEncoding("UTF-8").GetString(wc.DownloadData(string.Format(url)));
}
catch (Exception e)
{
throw new Exception(e.ToString());
}
}
}
public string ReadTxt() {
string Result = "";
try
{
string filename = "D:\\IPCACHE\\" + DateTime.Now.ToString("yyyyMMddHHmm") + ".txt";
using (StreamReader sr = new StreamReader(filename))
{
Result = sr.ReadToEnd();
}
}
catch
{
Result = "缓存不存在";
}
return Result;
}
public string WriteTxt(string val) {
try
{
string filename = "D:\\IPCACHE\\" + DateTime.Now.ToString("yyyyMMddHHmm") + ".txt";
using (StreamWriter sr = new StreamWriter(filename))
{
sr.Write(val);
}
}
catch
{
return "fail";
}
return "Success";
}
public string GetProxyIp(){
string ProxyApi = "代理IP获取的API, 结果应该是 0.0.0.0.0:xxxx";
CheckCacheDir();
string ip = "";
//检查缓存
string ProxyCache = ReadTxt();
if (ProxyCache != "缓存不存在" && ProxyCache != "")
{
ip = ProxyCache;
}
else {
ip = GetUrlContent(ProxyApi);
WriteTxt(ip);
}
return ip;
}
/// <summary>
/// 处理下载前的request
/// </summary>
/// <param name="response"></param>
public void BeforeRequest(RequestEntry request) {
//Console.WriteLine("BeforeRequest："+request.Url);
//request.Referer="";
//request.Headers.
Random r = new Random();
string Agent = agentList[r.Next(agentList.Length)];
request.Headers["User-Agent"] =Agent;
var proxy = new WebProxy(GetProxyIp(), true);
request.WebProxy=proxy;
}
/// <summary>
/// 处理下载完成后的http响应,网址、默认页、多页、内容分页
/// </summary>
/// <param name="response"></param>
public void AfterResponse(ResponseEntry response) {
Console.WriteLine("AfterResponse：" + response.Url);
}
}
}
评论