first commit

README.md

@@ -0,0 +1,30 @@
+# rotateproxy
+
+利用fofa搜索socks5开放代理进行代理池轮切的工具
+
+## 帮助
+
+```shell
+└──╼ #./cmd/rotateproxy/rotateproxy  -h
+Usage of ./cmd/rotateproxy/rotateproxy:
+  -email string
+    	email address
+  -l string
+    	listen address (default ":9")
+  -page int
+    	the page count you want to crawl (default 5)
+  -region int
+    	0: all 1: cannot bypass gfw 2: bypass gfw
+  -rule string
+    	search rule (default "protocol==\"socks5\" && \"Version:5 Method:No Authentication(0x00)\" && after=\"2021-11-01\"")
+  -time int
+    	Not used for more than milliseconds (default 8000)
+  -token string
+    	token
+
+```
+
+
+
+
+

check.go

@@ -0,0 +1,104 @@
+package rotateproxy
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type IPResponse struct {
+	ORIGIN    string     `json:"origin"`
+}
+
+
+type IPInfo struct {
+	Status      string  `json:"status"`
+	Country     string  `json:"country"`
+	CountryCode string  `json:"countryCode"`
+	Region      string  `json:"region"`
+	RegionName  string  `json:"regionName"`
+	City        string  `json:"city"`
+	Zip         string  `json:"zip"`
+	Lat         float64 `json:"lat"`
+	Lon         float64 `json:"lon"`
+	Timezone    string  `json:"timezone"`
+	Isp         string  `json:"isp"`
+	Org         string  `json:"org"`
+	As          string  `json:"as"`
+	Query       string  `json:"query"`
+}
+
+func CheckProxyAlive(proxyURL,ip string,timeout int) (respBody string, avail bool,time_config int) {
+	startTime := time.Now().UnixNano()
+	proxy, _ := url.Parse(proxyURL)
+	httpclient := &http.Client{
+		Transport: &http.Transport{
+			Proxy:           http.ProxyURL(proxy),
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Timeout: 20 * time.Second,
+	}
+	resp, err := httpclient.Get("http://httpbin.org/ip")
+	if err != nil {
+		return "", false,10000
+	}
+	defer resp.Body.Close()
+	var res IPResponse
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if len(res.ORIGIN)<2{
+		return "", false,10000
+	}
+	if err != nil {
+		return "", false,10000
+	}
+	endTime := time.Now().UnixNano()
+	Milliseconds:= int((endTime - startTime) / 1e6)// 毫秒
+	fmt.Println("IP : ",ip,"  用时毫秒 ",Milliseconds,"   可用")
+	if Milliseconds>timeout{
+		return "", false,10000
+	}
+	return res.ORIGIN, true,Milliseconds
+}
+
+func StartCheckProxyAlive(timeout int) {
+	go func() {
+		ticker := time.NewTicker(120 * time.Second)
+		for {
+			select {
+			case <-crawlDone:
+				fmt.Println("Checking")
+				checkAlive(timeout)
+				fmt.Println("Check done")
+			case <-ticker.C:
+				checkAlive(timeout)
+			}
+		}
+	}()
+}
+
+func checkAlive(timeout int) {
+	proxies, err := QueryProxyURL()
+	if err != nil {
+		fmt.Printf("[!] query db error: %v\n", err)
+	}
+	for i := range proxies {
+		proxy := proxies[i]
+		if proxy.Available{
+			continue
+		}
+		if proxy.Retry >5{
+			continue
+		}
+		go func() {
+			_, _,time_conig := CheckProxyAlive(proxy.URL,proxy.IP,timeout)
+			if time_conig !=10000 {
+				SetProxytime(proxy.URL, time_conig)
+			}else{
+				AddProxyURLRetry(proxy.URL)
+			}
+		}()
+	}
+}

cmd/rotateproxy/main.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/akkuman/rotateproxy"
+)
+
+var (
+	baseCfg   rotateproxy.BaseConfig
+	email     string
+	token     string
+	rule      string
+	pageCount int
+	timeout int
+)
+
+func init() {
+	flag.StringVar(&email, "email", "", "email address")
+	flag.StringVar(&token, "token", "", "token")
+	flag.StringVar(&baseCfg.ListenAddr, "l", ":9999", "listen address")
+	flag.IntVar(&timeout, "time", 8000, "Not used for more than milliseconds")
+	// && country="CN"
+	flag.StringVar(&rule, "rule", `protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="2021-11-01"`, "search rule")
+	flag.IntVar(&baseCfg.IPRegionFlag, "region", 0, "0: all 1: cannot bypass gfw 2: bypass gfw")
+	flag.IntVar(&pageCount, "page", 5, "the page count you want to crawl")
+	flag.Parse()
+}
+
+func isFlagPassed(name string) bool {
+	found := false
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == name {
+			found = true
+		}
+	})
+	return found
+}
+
+func main() {
+	if !isFlagPassed("email") || !isFlagPassed("token") {
+		flag.Usage()
+		return
+	}
+	rotateproxy.StartRunCrawler(token, email, rule, pageCount)
+	rotateproxy.StartCheckProxyAlive(timeout)
+	c := rotateproxy.NewRedirectClient(rotateproxy.WithConfig(&baseCfg))
+	c.Serve()
+	select {}
+}

crawler.go

@@ -0,0 +1,72 @@
+package rotateproxy
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+var crawlDone = make(chan struct{})
+
+type fofaAPIResponse struct {
+	Err     bool       `json:"error"`
+	Mode    string     `json:"mode"`
+	Page    int        `json:"page"`
+	Query   string     `json:"query"`
+	Results [][]string `json:"results"`
+	Size    int        `json:"size"`
+}
+
+func addProxyURL(url string) {
+	CreateProxyURL(url)
+}
+
+func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int) (err error) {
+	req, err := http.NewRequest("GET", "https://fofa.so/api/v1/search/all", nil)
+	if err != nil {
+		return err
+	}
+	rule = base64.StdEncoding.EncodeToString([]byte(rule))
+	q := req.URL.Query()
+	q.Add("email", fofaEmail)
+	q.Add("key", fofaApiKey)
+	q.Add("qbase64", rule)
+	q.Add("size", "500")
+	q.Add("page", fmt.Sprintf("%d", pageNum))
+	q.Add("fields", "host,title,ip,domain,port,country,city,server,protocol")
+	req.URL.RawQuery = q.Encode()
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("start to parse proxy url from response\n")
+	defer resp.Body.Close()
+	var res fofaAPIResponse
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("get %d host\n", len(res.Results))
+	for _, value := range res.Results {
+		host := value[0]
+		addProxyURL(fmt.Sprintf("socks5://%s", host))
+	}
+	crawlDone <- struct{}{}
+	return
+}
+
+func StartRunCrawler(fofaApiKey, fofaEmail, rule string, pageCount int) {
+	go func() {
+		for i := 1; i <= 3; i++ {
+			RunCrawler(fofaApiKey, fofaEmail, rule, i)
+		}
+		ticker := time.NewTicker(600 * time.Second)
+		for range ticker.C {
+			for i := 1; i <= 3; i++ {
+				RunCrawler(fofaApiKey, fofaEmail, rule, i)
+			}
+		}
+	}()
+}

db.go

@@ -0,0 +1,113 @@
+package rotateproxy
+
+import (
+	"fmt"
+	"regexp"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var DB *gorm.DB
+
+type ProxyURL struct {
+	gorm.Model
+	URL          string `gorm:"uniqueIndex;column:url"`
+	IP          string `gorm:"column:ip"`
+	CONSUMING     int `gorm:"column:time"`
+	COUNT 		int `gorm:"column:count"`
+	WEIGHT     int `gorm:"column:weight"`
+	Retry        int    `gorm:"column:retry"`
+	Available    bool   `gorm:"column:available"`
+	CanBypassGFW bool   `gorm:"column:can_bypass_gfw"`
+}
+
+func (ProxyURL) TableName() string {
+	return "proxy_urls"
+}
+
+func checkErr(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func init() {
+	var err error
+	DB, err = gorm.Open(sqlite.Open("db.db"), &gorm.Config{
+		Logger: logger.Discard,
+	})
+	checkErr(err)
+	DB.AutoMigrate(&ProxyURL{})
+}
+
+func CreateProxyURL(url string) error {
+	regstr := `\d+\.\d+\.\d+\.\d+`
+	reg, _ := regexp.Compile(regstr)
+	ip := reg.Find([]byte(url))
+	tx := DB.Create(&ProxyURL{
+		URL:       url,
+		IP:string(ip),
+		CONSUMING:0,
+		COUNT:0,
+		WEIGHT:1,
+		Retry:     0,
+		Available: false,
+	})
+	return tx.Error
+}
+
+func QueryAvailProxyURL() (proxyURLs []ProxyURL, err error) {
+	tx := DB.Where("available = ?", true).Find(&proxyURLs)
+	err = tx.Error
+	return
+}
+
+func QueryProxyURL() (proxyURLs []ProxyURL, err error) {
+	tx := DB.Find(&proxyURLs)
+	err = tx.Error
+	return
+}
+
+func SetProxyURLAvail(url string, canBypassGFW bool) error {
+	tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Updates(ProxyURL{Retry: 0, Available: true, CanBypassGFW: canBypassGFW})
+	return tx.Error
+}
+func StopProxy(url string) error {
+	tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Update("Available",false)
+	return tx.Error
+}
+
+func SetProxytime(url string,timeconfig int) error {
+	tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Updates(ProxyURL{CONSUMING: timeconfig,Available: true})
+	return tx.Error
+}
+
+func AddProxyURLRetry(url string) error {
+	tx := DB.Model(&ProxyURL{}).Where("url = ?", url).Update("retry", gorm.Expr("retry + 1"))
+	return tx.Error
+}
+
+
+func RandomProxyURL(regionFlag int) (string, error) {
+	var proxyURL ProxyURL
+	var tx *gorm.DB
+	switch regionFlag {
+	case 1:
+		tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = ? AND can_bypass_gfw = ? ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName()), true, false).Scan(&proxyURL)
+	case 2:
+		tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = ? AND can_bypass_gfw = ? ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName()), true, true).Scan(&proxyURL)
+	default:
+		tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = 1 ORDER BY RANDOM() LIMIT 1;", proxyURL.TableName())).Scan(&proxyURL)
+	}
+	return proxyURL.URL, tx.Error
+}
+
+
+func RandomProxyURL2() (string, error) {
+	var proxyURL ProxyURL
+	var tx *gorm.DB
+	tx = DB.Raw(fmt.Sprintf("SELECT * FROM %s WHERE available = 1 ORDER BY time ASC LIMIT 1;", proxyURL.TableName())).Scan(&proxyURL)
+	return proxyURL.URL, tx.Error
+}

go.mod

@@ -0,0 +1,14 @@
+module github.com/akkuman/rotateproxy
+
+go 1.17
+
+require (
+	gorm.io/driver/sqlite v1.1.6
+	gorm.io/gorm v1.21.16
+)
+
+require (
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.2 // indirect
+	github.com/mattn/go-sqlite3 v1.14.8 // indirect
+)

makefile

@@ -0,0 +1,2 @@
+build:
+	cd cmd/rotateproxy && go build -trimpath -ldflags="-s -w"