Improve concurrency

README.md

@@ -3,7 +3,7 @@
   <img alt="Jaeles" src="https://image.flaticon.com/icons/svg/1432/1432425.svg" height="140" />
   <p align="center">
     <a href=""><img alt="Software License" src="https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square"></a>
-    <a href="https://github.com/jaeles-project/jaeles"><img alt="Release" src="https://img.shields.io/badge/version-beta%20v0.5-blue.svg"></a>
+    <a href="https://github.com/jaeles-project/jaeles"><img alt="Release" src="https://img.shields.io/badge/version-beta%20v0.6-blue.svg"></a>
     <a href="https://inventory.rawsec.ml/tools.html#Jaeles"><img src="https://inventory.rawsec.ml/img/badges/Rawsec-inventoried-FF5050_flat.svg" alt="Rawsec&#39;s CyberSecurity Inventory"></a>
   </p>
 </p>
@@ -37,6 +37,8 @@ jaeles scan -c 50 -s '/tmp/custom-signature/.*' -U list_of_urls.txt
 cat urls.txt | grep 'interesting' | jaeles scan -c 50 -s 'fuzz/.*' -U list_of_urls.txt --proxy http://127.0.0.1:8080
 
 jaeles server --verbose -s sqli
+
+jaeles scan -v -s '/tmp/sensitive/.*' -u 'https://wp.target.com' -p 'root=[[.URL]]'
 ```
 
 More usage can be found [here](https://jaeles-project.github.io/usage/)

cmd/config.go

@@ -12,8 +12,6 @@ import (
 	"path/filepath"
 )
 
-var configCmd *cobra.Command
-
 func init() {
 	// configCmd represents the config command
 	var configCmd = &cobra.Command{
@@ -39,7 +37,7 @@ func init() {
 
 }
 
-func runConfig(cmd *cobra.Command, args []string) error {
+func runConfig(cmd *cobra.Command, _ []string) error {
 	// print more help
 	helps, _ := cmd.Flags().GetBool("hh")
 	if helps == true {

cmd/root.go

@@ -56,7 +56,7 @@ func init() {
 
 	RootCmd.PersistentFlags().BoolVarP(&options.EnablePassive, "passive", "G", false, "Turn on passive detections")
 	RootCmd.PersistentFlags().StringVar(&options.SelectedPassive, "sp", "*", "Selector for passive detections")
-	RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 20, "concurrency")
+	RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 20, "Set the concurrency level")
 	RootCmd.PersistentFlags().StringVarP(&options.Output, "output", "o", "out", "output folder name")
 	RootCmd.PersistentFlags().StringVar(&options.PassiveOutput, "passiveOutput", "", "Passive output folder (default is passive-out)")
 	RootCmd.PersistentFlags().StringVar(&options.PassiveSummary, "passiveSummary", "", "Passive Summary file")

cmd/scan.go

@@ -3,20 +3,18 @@ package cmd
 import (
 	"bufio"
 	"fmt"
+	"github.com/jaeles-project/jaeles/core"
 	"github.com/jaeles-project/jaeles/libs"
 	"github.com/jaeles-project/jaeles/sender"
 	"github.com/jaeles-project/jaeles/utils"
+	"github.com/panjf2000/ants"
+	"github.com/spf13/cobra"
 	"github.com/thoas/go-funk"
 	"os"
 	"strings"
 	"sync"
-
-	"github.com/jaeles-project/jaeles/core"
-	"github.com/spf13/cobra"
 )
 
-var scanCmd *cobra.Command
-
 func init() {
 	// scanCmd represents the scan command
 	var scanCmd = &cobra.Command{
@@ -33,7 +31,7 @@ func init() {
 	RootCmd.AddCommand(scanCmd)
 }
 
-func runScan(cmd *cobra.Command, args []string) error {
+func runScan(cmd *cobra.Command, _ []string) error {
 	SelectSign()
 	var urls []string
 	// parse URL input here
@@ -80,7 +78,7 @@ func runScan(cmd *cobra.Command, args []string) error {
 		OriginRaw = core.ParseBurpRequest(RawRequest)
 	}
 
-	// Really start do something
+	/* ---- Really start do something ---- */
 
 	// run background detector
 	if !options.NoBackGround {
@@ -91,30 +89,7 @@ func runScan(cmd *cobra.Command, args []string) error {
 		}()
 	}
 
-	jobs := make(chan libs.Job)
-
 	var wg sync.WaitGroup
-	for i := 0; i < options.Concurrency; i++ {
-		wg.Add(1)
-		go func() {
-			for job := range jobs {
-				sign := job.Sign
-				url := job.URL
-
-				// get origin from -r req.txt options
-				if OriginRaw.Raw != "" {
-					sign.Origin = OriginRaw
-				}
-				if RawRequest != "" {
-					sign.RawRequest = RawRequest
-				}
-				RunJob(url, sign, options)
-			}
-			wg.Done()
-		}()
-	}
-
-	// jobs to send request
 	for _, signFile := range options.SelectedSigns {
 		sign, err := core.ParseSign(signFile)
 		if err != nil {
@@ -125,16 +100,37 @@ func runScan(cmd *cobra.Command, args []string) error {
 		if sign.Level > options.Level {
 			continue
 		}
+
+		p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+			startScanJob(i)
+			wg.Done()
+		})
+		defer p.Release()
+
+		//get origin from -r req.txt options
+		if OriginRaw.Raw != "" {
+			sign.Origin = OriginRaw
+		}
+		if RawRequest != "" {
+			sign.RawRequest = RawRequest
+		}
+
+		// Submit tasks one by one.
 		for _, url := range urls {
-			jobs <- libs.Job{url, sign}
+			wg.Add(1)
+			job := libs.Job{url, sign}
+			_ = p.Invoke(job)
 		}
 	}
-
-	close(jobs)
 	wg.Wait()
 	return nil
 }
 
+func startScanJob(j interface{}) {
+	job := j.(libs.Job)
+	RunJob(job.URL, job.Sign, options)
+}
+
 // RunJob really run the job
 func RunJob(url string, sign libs.Signature, options libs.Options) {
 	var originRec libs.Record

cmd/server.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/panjf2000/ants"
 	"path"
 	"path/filepath"
 	"sync"
@@ -15,8 +16,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var serverCmd *cobra.Command
-
 func init() {
 	// serverCmd represents the server command
 	var serverCmd = &cobra.Command{
@@ -30,7 +29,7 @@ func init() {
 
 }
 
-func runServer(cmd *cobra.Command, args []string) error {
+func runServer(cmd *cobra.Command, _ []string) error {
 	SelectSign()
 	// prepare DB stuff
 	if options.Server.Username != "" {
@@ -46,9 +45,14 @@ func runServer(cmd *cobra.Command, args []string) error {
 	}
 	database.InitConfigSign()
 
-	result := make(chan libs.Record)
-	jobs := make(chan libs.Job)
+	var wg sync.WaitGroup
+	p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
+		startScanJob(i)
+		wg.Done()
+	})
+	defer p.Release()
 
+	result := make(chan libs.Record)
 	go func() {
 		for {
 			record := <-result
@@ -67,7 +71,10 @@ func runServer(cmd *cobra.Command, args []string) error {
 				// parse sign as list or single
 				if sign.Type != "fuzz" {
 					url := record.OriginReq.URL
-					jobs <- libs.Job{URL: url, Sign: sign}
+					//jobs <- libs.Job{URL: url, Sign: sign}
+					wg.Add(1)
+					job := libs.Job{url, sign}
+					_ = p.Invoke(job)
 				} else {
 					fuzzSign := sign
 					fuzzSign.Requests = []libs.Request{}
@@ -81,34 +88,23 @@ func runServer(cmd *cobra.Command, args []string) error {
 						fuzzSign.Requests = append(fuzzSign.Requests, req)
 					}
 					url := record.OriginReq.URL
-					jobs <- libs.Job{URL: url, Sign: fuzzSign}
 
+					wg.Add(1)
+					job := libs.Job{url, sign}
+					_ = p.Invoke(job)
 				}
 			}
 
 		}
 	}()
 
-	/* Start sending request here */
-	var wg sync.WaitGroup
-	for i := 0; i < options.Concurrency; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for job := range jobs {
-				sign := job.Sign
-				url := job.URL
-				RunJob(url, sign, options)
-			}
-		}()
-	}
-
 	host, _ := cmd.Flags().GetString("host")
 	port, _ := cmd.Flags().GetString("port")
 	bind := fmt.Sprintf("%v:%v", host, port)
 	options.Server.Bind = bind
 	utils.InforF("Start API server at %v", fmt.Sprintf("http://%v/#/", bind))
 
 	server.InitRouter(options, result)
+	wg.Wait()
 	return nil
 }

core/background.go

@@ -16,6 +16,7 @@ import (
 func Background(options libs.Options) {
 	utils.DebugF("Checking backround task")
 	time.Sleep(time.Duration(options.Refresh) * time.Second)
+	// currently disable for now
 	PollingLog()
 	PickupLog(options)
 	// @TODO: Add passive signature for analyzer each request

core/detecter.go

@@ -78,8 +78,12 @@ func RunDetector(record libs.Record, detectionString string) (string, bool) {
 	})
 
 	vm.Set("StringSearch", func(call otto.FunctionCall) otto.Value {
-		componentName := call.Argument(0).String()
-		analyzeString := call.Argument(1).String()
+		args := call.ArgumentList
+		componentName := "response"
+		analyzeString := args[0].String()
+		if len(args) >= 2 {
+			analyzeString = args[1].String()
+		}
 		component := GetComponent(record, componentName)
 		validate := StringSearch(component, analyzeString)
 		result, _ := vm.ToValue(validate)
@@ -96,8 +100,12 @@ func RunDetector(record libs.Record, detectionString string) (string, bool) {
 	})
 
 	vm.Set("RegexSearch", func(call otto.FunctionCall) otto.Value {
-		componentName := call.Argument(0).String()
-		analyzeString := call.Argument(1).String()
+		args := call.ArgumentList
+		componentName := "response"
+		analyzeString := args[0].String()
+		if len(args) >= 2 {
+			analyzeString = args[1].String()
+		}
 		component := GetComponent(record, componentName)
 		matches, validate := RegexSearch(component, analyzeString)
 		result, err := vm.ToValue(validate)
@@ -261,6 +269,7 @@ func GetComponent(record libs.Record, component string) string {
 
 // StringSearch search string literal in component
 func StringSearch(component string, analyzeString string) bool {
+	utils.DebugF("analyzeString: %v", analyzeString)
 	if strings.Contains(component, analyzeString) {
 		return true
 	}
@@ -274,6 +283,7 @@ func StringCount(component string, analyzeString string) int {
 
 // RegexSearch search regex string in component
 func RegexSearch(component string, analyzeString string) (string, bool) {
+	utils.DebugF("analyzeString: %v", analyzeString)
 	var result bool
 	var extra string
 	r, err := regexp.Compile(analyzeString)

core/signature.go

@@ -154,7 +154,6 @@ func ResolveVariable(format string, data map[string]string) string {
 	if strings.TrimSpace(format) == "" {
 		return format
 	}
-
 	_, exist := data["original"]
 	if !exist {
 		data["original"] = ""
@@ -197,7 +196,6 @@ func AltResolveVariable(format string, data map[string]string) string {
 		return format
 	}
 	realFormat, err := template.New("").Delims("[[", "]]").Parse(format)
-
 	_, exist := data["original"]
 	if !exist {
 		data["original"] = ""

core/update.go

@@ -50,6 +50,7 @@ func UpdateSignature(options libs.Options, customRepo string) {
 		os.RemoveAll(signPath)
 		os.RemoveAll(options.PassiveFolder)
 		os.RemoveAll(options.ResourcesFolder)
+		os.RemoveAll(options.ThirdPartyFolder)
 	}
 	if options.Server.Key != "" {
 		cmd := fmt.Sprintf("GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -i %v' git clone --depth=1 %v %v", options.Server.Key, url, signPath)

go.mod

@@ -14,16 +14,17 @@ require (
 	github.com/gin-gonic/gin v1.5.0
 	github.com/go-resty/resty/v2 v2.2.0
 	github.com/google/uuid v1.1.1
-	github.com/gorilla/websocket v1.4.1
+	github.com/gorilla/websocket v1.4.2
 	github.com/jinzhu/gorm v1.9.12
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/panjf2000/ants v1.3.0
 	github.com/parnurzeal/gorequest v0.2.16
 	github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.6
 	github.com/spf13/viper v1.6.2
-	github.com/thoas/go-funk v0.5.0
+	github.com/thoas/go-funk v0.6.0
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1

go.sum

@@ -95,8 +95,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
-github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -158,6 +158,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M=
+github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY=
 github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
 github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
@@ -213,8 +215,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/thoas/go-funk v0.5.0 h1:XXFUVqX6xnIDqXxENFHBFS1X5AoT0EDs7HJq2krRfD8=
-github.com/thoas/go-funk v0.5.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/thoas/go-funk v0.6.0 h1:ryxN0pa9FnI7YHgODdLIZ4T6paCZJt8od6N9oRztMxM=
+github.com/thoas/go-funk v0.6.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

libs/version.go

@@ -2,7 +2,7 @@ package libs
 
 const (
 	// VERSION current Jaeles version
-	VERSION = "beta v0.5.6"
+	VERSION = "beta v0.5.7"
 	// AUTHOR author of this
 	AUTHOR = "@j3ssiejjj"
 	// SIGNREPO default repo to get signature