Merge pull request #145 from edoardottt/devel

v1.3.3

README.md

@@ -10,33 +10,6 @@
   <a href="https://github.com/edoardottt/cariddi/actions">
     <img src="https://github.com/edoardottt/cariddi/workflows/Go/badge.svg?branch=main" alt="workflows" />
   </a>
-  <!-- ubuntu-build -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/ubuntu-build.svg" alt="ubuntu-build" />
-  </a>
-  <!-- win10-build -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/win10.svg" alt="win10-build" />
-  </a>
-  <!-- pr-welcome -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/pr-welcome.svg" alt="pr-welcome" />
-  </a>
-
-  <br>
-
-  <!-- mainteinance -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/maintained-yes.svg" alt="Mainteinance yes" />
-  </a>
-  <!-- ask-me-anything -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/ask-me-anything.svg" alt="ask me anything" />
-  </a>
-  <!-- license GPLv3.0 -->
-  <a href="https://github.com/edoardottt/cariddi/blob/master/LICENSE">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/license-GPL3.svg" alt="license-GPL3" />
-  </a>
   <br>
   <sub>
     Coded with 💙 by edoardottt

internal/file/file.go

@@ -97,17 +97,18 @@ func CreateOutputFile(target string, subcommand string, format string) string {
 		f.Close()
 	} else {
 		// The file already exists, overwrite.
-
 		f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
 		if err != nil {
 			fmt.Println("Can't create output file.")
 			os.Exit(1)
 		}
+
 		err = f.Truncate(0)
 		if err != nil {
 			fmt.Println("Can't create output file.")
 			os.Exit(1)
 		}
+
 		f.Close()
 	}
 

pkg/crawler/colly.go

@@ -33,6 +33,7 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"strings"
@@ -256,6 +257,7 @@ func New(scan *Scan) *Results {
 			if !scan.Plain {
 				fmt.Fprint(os.Stdout, "\r")
 				fmt.Println("CTRL+C pressed: Exiting")
+
 				cCount++
 			}
 
@@ -317,16 +319,23 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
 
 	// Use a Proxy if needed
 	if proxy != "" {
-		err := c.SetProxy(proxy)
+		proxyParsed, err := url.Parse(proxy)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
 		}
-	}
 
-	c.WithTransport(&http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	})
+		c.WithTransport(&http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true},
+			Proxy:             http.ProxyURL(proxyParsed),
+			DisableKeepAlives: true,
+		})
+	} else {
+		c.WithTransport(&http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		})
+	}
 
 	return c
 }

pkg/crawler/requests.go

@@ -61,13 +61,14 @@ func GetRequest(target string) (string, error) {
 func PostRequest(target string, data map[string]string) (string, error) {
 	postBody, _ := json.Marshal(data)
 	responseBody := bytes.NewBuffer(postBody)
-	// Leverage Go's HTTP Post function to make request
+
 	resp, err := http.Post(target, "application/json", responseBody)
-	// Handle Error
 	if err != nil {
 		return "", err
 	}
+
 	defer resp.Body.Close()
+
 	// Read the response body
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {

pkg/crawler/scan.go

@@ -73,6 +73,7 @@ func SecretsMatch(url, body string, secretsFile *[]string) []scanner.SecretMatch
 			if matched, err := regexp.Match(secret, []byte(body)); err == nil && matched {
 				re := regexp.MustCompile(secret)
 				matches := re.FindAllStringSubmatch(body, -1)
+
 				for _, match := range matches {
 					secretScanned := scanner.Secret{Name: "CustomFromFile", Description: "", Regex: secret, Poc: ""}
 					secretFound := scanner.SecretMatched{Secret: secretScanned, URL: url, Match: match[0]}

pkg/output/banner.go

@@ -35,7 +35,7 @@ import (
 
 // nolint: checknoglobals
 const (
-	version = "v1.3.2"
+	version = "v1.3.3"
 	banner  = `                 _     _     _ _ 
                 (_)   | |   | (_)
    ___ __ _ _ __ _  __| | __| |_ 

pkg/scanner/info.go

@@ -61,17 +61,19 @@ func GetInfoRegexes() []Info {
 			"IPv4 address",
 			`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`,
 		},
-		{
-			"BTC address",
-			`([13]|bc1)[A-HJ-NP-Za-km-z1-9]{27,34}`,
-		},
+		/*
+			TOO MANY FALSE POSITIVES
+			{
+				"BTC address",
+				`([13]|bc1)[A-HJ-NP-Za-km-z1-9]{27,34}`,
+			},
+		*/
 		/*
 			HOW TO AVOID VERY VERY LONG BASE64 IMAGES ???
-				{
-					"Base64-encoded JSON",
-					[]string{
-						`ey(A|B)[A-Za-z0-9+\/]{20,}(={0,2})`},
-				},
+			{
+				"Base64-encoded JSON",
+				`ey(A|B)[A-Za-z0-9+\/]{20,}(={0,2})`,
+			},
 		*/
 	}
 

snapcraft.yaml

@@ -2,7 +2,7 @@ name: cariddi
 summary: Fast web crawler and scanner
 description: |
       Take a list of domains, crawl urls and scan for endpoints, secrets, api keys, file extensions, tokens and more
-version: 1.3.2
+version: 1.3.3
 grade: stable
 base: core20