diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4b0955 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# rotateproxy + +利用fofa搜索socks5开放代理进行代理池轮切的工具 + +## 帮助 + +```shell +└──╼ #./cmd/rotateproxy/rotateproxy -h +Usage of ./cmd/rotateproxy/rotateproxy: + -email string + email address + -l string + listen address (default ":9") + -page int + the page count you want to crawl (default 5) + -region int + 0: all 1: cannot bypass gfw 2: bypass gfw + -rule string + search rule (default "protocol==\"socks5\" && \"Version:5 Method:No Authentication(0x00)\" && after=\"2021-11-01\"") + -time int + Not used for more than milliseconds (default 8000) + -token string + token + +``` + + + + + diff --git a/check.go b/check.go new file mode 100644 index 0000000..fcdd1e1 --- /dev/null +++ b/check.go @@ -0,0 +1,104 @@ +package rotateproxy + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" +) + +type IPResponse struct { + ORIGIN string `json:"origin"` +} + + +type IPInfo struct { + Status string `json:"status"` + Country string `json:"country"` + CountryCode string `json:"countryCode"` + Region string `json:"region"` + RegionName string `json:"regionName"` + City string `json:"city"` + Zip string `json:"zip"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Timezone string `json:"timezone"` + Isp string `json:"isp"` + Org string `json:"org"` + As string `json:"as"` + Query string `json:"query"` +} + +func CheckProxyAlive(proxyURL,ip string,timeout int) (respBody string, avail bool,time_config int) { + startTime := time.Now().UnixNano() + proxy, _ := url.Parse(proxyURL) + httpclient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxy), + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Timeout: 20 * time.Second, + } + resp, err := httpclient.Get("http://httpbin.org/ip") + if err != nil { + return "", false,10000 + } + defer resp.Body.Close() + var res IPResponse + err = json.NewDecoder(resp.Body).Decode(&res) + if len(res.ORIGIN)<2{ + return "", false,10000 + } + if err != nil { + return "", false,10000 + } + endTime := time.Now().UnixNano() + Milliseconds:= int((endTime - startTime) / 1e6)// 毫秒 + fmt.Println("IP : ",ip," 用时毫秒 ",Milliseconds," 可用") + if Milliseconds>timeout{ + return "", false,10000 + } + return res.ORIGIN, true,Milliseconds +} + +func StartCheckProxyAlive(timeout int) { + go func() { + ticker := time.NewTicker(120 * time.Second) + for { + select { + case <-crawlDone: + fmt.Println("Checking") + checkAlive(timeout) + fmt.Println("Check done") + case <-ticker.C: + checkAlive(timeout) + } + } + }() +} + +func checkAlive(timeout int) { + proxies, err := QueryProxyURL() + if err != nil { + fmt.Printf("[!] query db error: %v\n", err) + } + for i := range proxies { + proxy := proxies[i] + if proxy.Available{ + continue + } + if proxy.Retry >5{ + continue + } + go func() { + _, _,time_conig := CheckProxyAlive(proxy.URL,proxy.IP,timeout) + if time_conig !=10000 { + SetProxytime(proxy.URL, time_conig) + }else{ + AddProxyURLRetry(proxy.URL) + } + }() + } +} diff --git a/cmd/rotateproxy/main.go b/cmd/rotateproxy/main.go new file mode 100644 index 0000000..67e4e46 --- /dev/null +++ b/cmd/rotateproxy/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "flag" + + "github.com/akkuman/rotateproxy" +) + +var ( + baseCfg rotateproxy.BaseConfig + email string + token string + rule string + pageCount int + timeout int +) + +func init() { + flag.StringVar(&email, "email", "", "email address") + flag.StringVar(&token, "token", "", "token") + flag.StringVar(&baseCfg.ListenAddr, "l", ":9999", "listen address") + flag.IntVar(&timeout, "time", 8000, "Not used for more than milliseconds") + // && country="CN" + flag.StringVar(&rule, "rule", `protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="2021-11-01"`, "search rule") + flag.IntVar(&baseCfg.IPRegionFlag, "region", 0, "0: all 1: cannot bypass gfw 2: bypass gfw") + flag.IntVar(&pageCount, "page", 5, "the page count you want to crawl") + flag.Parse() +} + +func isFlagPassed(name string) bool { + found := false + flag.Visit(func(f *flag.Flag) { + if f.Name == name { + found = true + } + }) + return found +} + +func main() { + if !isFlagPassed("email") || !isFlagPassed("token") { + flag.Usage() + return + } + rotateproxy.StartRunCrawler(token, email, rule, pageCount) + rotateproxy.StartCheckProxyAlive(timeout) + c := rotateproxy.NewRedirectClient(rotateproxy.WithConfig(&baseCfg)) + c.Serve() + select {} +} diff --git a/crawler.go b/crawler.go new file mode 100644 index 0000000..1ebf2a3 --- /dev/null +++ b/crawler.go @@ -0,0 +1,72 @@ +package rotateproxy + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "time" +) + +var crawlDone = make(chan struct{}) + +type fofaAPIResponse struct { + Err bool `json:"error"` + Mode string `json:"mode"` + Page int `json:"page"` + Query string `json:"query"` + Results [][]string `json:"results"` + Size int `json:"size"` +} + +func addProxyURL(url string) { + CreateProxyURL(url) +} + +func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int) (err error) { + req, err := http.NewRequest("GET", "https://fofa.so/api/v1/search/all", nil) + if err != nil { + return err + } + rule = base64.StdEncoding.EncodeToString([]byte(rule)) + q := req.URL.Query() + q.Add("email", fofaEmail) + q.Add("key", fofaApiKey) + q.Add("qbase64", rule) + q.Add("size", "500") + q.Add("page", fmt.Sprintf("%d", pageNum)) + q.Add("fields", "host,title,ip,domain,port,country,city,server,protocol") + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + fmt.Printf("start to parse proxy url from response\n") + defer resp.Body.Close() + var res fofaAPIResponse + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + return err + } + fmt.Printf("get %d host\n", len(res.Results)) + for _, value := range res.Results { + host := value[0] + addProxyURL(fmt.Sprintf("socks5://%s", host)) + } + crawlDone <- struct{}{} + return +} + +func StartRunCrawler(fofaApiKey, fofaEmail, rule string, pageCount int) { + go func() { + for i := 1; i <= 3; i++ { + RunCrawler(fofaApiKey, fofaEmail, rule, i) + } + ticker := time.NewTicker(600 * time.Second) + for range ticker.C { + for i := 1; i <= 3; i++ { + RunCrawler(fofaApiKey, fofaEmail, rule, i) + } + } + }() +} diff --git a/db.go b/db.go new file mode 100644 index 0000000..1e91548 --- /dev/null +++ b/db.go @@ -0,0 +1,113 @@ +package rotateproxy + +import ( + "fmt" + "regexp" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var DB *gorm.DB + +type ProxyURL struct { + gorm.Model + URL string `gorm:"uniqueIndex;column:url"` + IP string `gorm:"column:ip"` + CONSUMING int `gorm:"column:time"` + COUNT int `gorm:"column:count"` + WEIGHT int `gorm:"column:weight"` + Retry int `gorm:"column:retry"` + Available bool `gorm:"column:available"` + CanBypassGFW bool `gorm:"column:can_bypass_gfw"` +} + +func (ProxyURL) TableName() string { + return "proxy_urls" +} + +func checkErr(err error) { + if err != nil { + panic(err) + } +} + +func init() { + var err error + DB, err = gorm.Open(sqlite.Open("db.db"), &gorm.Config{ + Logger: logger.Discard, + }) + checkErr(err) + DB.AutoMigrate(&ProxyURL{}) +} + +func CreateProxyURL(url string) error { + regstr := `\d+\.\d+\.\d+\.\d+` + reg, _ := regexp.Compile(regstr) + ip := reg.Find([]byte(url)) + tx := DB.Create(&ProxyURL{ + URL: url, + IP:string(ip), + CONSUMING:0, + COUNT:0, + WEIGHT:1, + Retry: 0, + Available: false, + }) + return tx.Error +} + +func QueryAvailProxyURL() (proxyURLs []ProxyURL, err error) { + tx := DB.Where("available = ?", true).Find(&proxyURLs) + err = tx.Error + return +} + +func QueryProxyURL() (proxyURLs []ProxyURL, err error) { + tx := DB.Find(&proxyURLs) + err = tx.Error + return +} + +func SetProxyURLAvail(url string, canBypassGFW bool) error { + tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Updates(ProxyURL{Retry: 0, Available: true, CanBypassGFW: canBypassGFW}) + return tx.Error +} +func StopProxy(url string) error { + tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Update("Available",false) + return tx.Error +} + +func SetProxytime(url string,timeconfig int) error { + tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Updates(ProxyURL{CONSUMING: timeconfig,Available: true}) + return tx.Error +} + +func AddProxyURLRetry(url string) error { + tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Update("retry", gorm.Expr("retry + 1")) + return tx.Error +} + + +func RandomProxyURL(regionFlag int) (string, error) { + var proxyURL ProxyURL + var tx *gorm.DB + switch regionFlag { + case 1: + tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = ? AND can_bypass_gfw = ? ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName()), true, false).Scan(&proxyURL) + case 2: + tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = ? AND can_bypass_gfw = ? ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName()), true, true).Scan(&proxyURL) + default: + tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = 1 ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName())).Scan(&proxyURL) + } + return proxyURL.URL, tx.Error +} + + +func RandomProxyURL2() (string, error) { + var proxyURL ProxyURL + var tx *gorm.DB + tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = 1 ORDER BY time ASC LIMIT 1;", proxyURL.TableName())).Scan(&proxyURL) + return proxyURL.URL, tx.Error +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6500ad4 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module github.com/akkuman/rotateproxy + +go 1.17 + +require ( + gorm.io/driver/sqlite v1.1.6 + gorm.io/gorm v1.21.16 +) + +require ( + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.2 // indirect + github.com/mattn/go-sqlite3 v1.14.8 // indirect +) diff --git a/makefile b/makefile new file mode 100644 index 0000000..73a6390 --- /dev/null +++ b/makefile @@ -0,0 +1,2 @@ +build: + cd cmd/rotateproxy && go build -trimpath -ldflags="-s -w" \ No newline at end of file diff --git a/pics/curl-run.jpg b/pics/curl-run.jpg new file mode 100644 index 0000000..e0d28d2 Binary files /dev/null and b/pics/curl-run.jpg differ diff --git a/traffic_redirect.go b/traffic_redirect.go new file mode 100644 index 0000000..90e4df5 --- /dev/null +++ b/traffic_redirect.go @@ -0,0 +1,212 @@ +package rotateproxy + +import ( + "fmt" + "io" + "net" + "strings" + "time" +) + +var ( + largeBufferSize = 32 * 1024 // 32KB large buffer +) + +type BaseConfig struct { + ListenAddr string + IPRegionFlag int // 0: all 1: cannot bypass gfw 2: bypass gfw +} + +type RedirectClient struct { + config *BaseConfig +} + +type RedirectClientOption func(*RedirectClient) + +func WithConfig(config *BaseConfig) RedirectClientOption { + return func(c *RedirectClient) { + c.config = config + } +} + +func NewRedirectClient(opts ...RedirectClientOption) *RedirectClient { + c := &RedirectClient{} + for _, opt := range opts { + opt(c) + } + return c +} + +func (c *RedirectClient) Serve() error { + l, err := net.Listen("tcp", c.config.ListenAddr) + if err != nil { + return err + } + for IsProxyURLBlank() { + fmt.Println("[*] waiting for crawl proxy...") + time.Sleep(3 * time.Second) + } + for { + conn, err := l.Accept() + if err != nil { + fmt.Printf("[!] accept error: %v\n", err) + continue + } + go c.HandleConn(conn) + } +} + + +func (c *RedirectClient) HandleConn(conn net.Conn) { + startTime := time.Now().UnixNano() + key, err := RandomProxyURL(c.config.IPRegionFlag) + if err != nil { + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error : %v\n", errConn) + } + return + } + key2 := strings.TrimPrefix(key, "socks5://") + fmt.Println(key) + cc, err := net.DialTimeout("tcp", key2, 20*time.Second) + if err != nil { + //如果超时。则踢出这个节点 + StopProxy(key) + fmt.Printf("[!] cannot connect to error1 %v\n", key2) + closeConn(conn) + return + } + endTime := time.Now().UnixNano() + Milliseconds:= int((endTime - startTime) / 1e6)// 毫秒 + if Milliseconds>30000{ + fmt.Printf("[!] cannot connect to error2 %v\n", key2) + StopProxy(key) + closeConn(conn) + return + } + go func() { + err = transport(conn, cc) + if err != nil { + fmt.Printf("[!] connect error: %v\n", err) + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error: %v\n", errConn) + } + errConn = closeConn(cc) + if errConn != nil { + fmt.Printf("[!] close upstream connect error: %v\n", errConn) + } + } + }() + +} + +func (c *RedirectClient) HandleConn2(conn net.Conn) { + key, err := RandomProxyURL(c.config.IPRegionFlag) + if err != nil { + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error: %v\n", errConn) + } + return + } + key2 := strings.TrimPrefix(key, "socks5://") + fmt.Println(key) + cc, err := net.DialTimeout("tcp", key2, 10*time.Second) + flag:=true + if err != nil { + //如果超时。则踢出这个节点 + StopProxy(key) + flag=flag + fmt.Printf("[!] cannot connect to error %v\n", key2) + } + if !flag{ + url2, err := RandomProxyURL2() + if err != nil { + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error2: %v\n", errConn) + } + return + } + key2 := strings.TrimPrefix(url2, "socks5://") + fmt.Println(key) + cc, err := net.DialTimeout("tcp", key2, 5*time.Second) + if err != nil { + //如果超时。则踢出这个节点 + StopProxy(key) + flag=flag + fmt.Printf("[!] cannot connect111 to %v\n", key2) + } + go func() { + err = transport(conn, cc) + if err != nil { + fmt.Printf("[!] connect error222: %v\n", err) + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error333: %v\n", errConn) + } + errConn = closeConn(cc) + if errConn != nil { + fmt.Printf("[!] close upstream connect error44: %v\n", errConn) + } + } + }() + + }else{ + go func() { + err = transport(conn, cc) + if err != nil { + fmt.Printf("[!] connect error: %v\n", err) + errConn := closeConn(conn) + if errConn != nil { + fmt.Printf("[!] close connect error: %v\n", errConn) + } + errConn = closeConn(cc) + if errConn != nil { + fmt.Printf("[!] close upstream connect error: %v\n", errConn) + } + } + }() + } +} + +func closeConn(conn net.Conn) (err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("%v", e) + } + }() + err = conn.Close() + return err +} + +func transport(rw1, rw2 io.ReadWriter) error { + errc := make(chan error, 1) + go func() { + errc <- copyBuffer(rw1, rw2) + }() + + go func() { + errc <- copyBuffer(rw2, rw1) + }() + + err := <-errc + if err != nil && err == io.EOF { + err = nil + } + return err +} + +func copyBuffer(dst io.Writer, src io.Reader) (err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("%v", e) + } + }() + buf := make([]byte, largeBufferSize) + + _, err = io.CopyBuffer(dst, src, buf) + return err +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..4f0e9e4 --- /dev/null +++ b/util.go @@ -0,0 +1,33 @@ +package rotateproxy + +import ( + "fmt" + "math/rand" + "sync" + "time" +) + +func init() { + rand.Seed(time.Now().Unix()) +} + +func RandomSyncMap(sMap sync.Map) (key, value interface{}) { + var tmp [][2]interface{} + sMap.Range(func(key, value interface{}) bool { + if value.(int) == 0 { + tmp = append(tmp, [2]interface{}{key, value}) + } + return true + }) + element := tmp[rand.Intn(len(tmp))] + return element[0], element[1] +} + +func IsProxyURLBlank() bool { + proxies, err := QueryAvailProxyURL() + if err != nil { + fmt.Printf("[!] Error: %v\n", err) + return false + } + return len(proxies) == 0 +}