Merge remote-tracking branch 'origin/main' into wdg/various

.github/workflows/test.yaml

@@ -0,0 +1,29 @@
+name: Go Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: Run Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.20'
+
+      - name: Install dependencies
+        run: go mod download
+
+      - name: Run tests
+        run: go test -v ./...

corediff.go

@@ -136,7 +136,7 @@ func checkPath(root string, db hashDB, args *baseArgs) (stats *walkStats) {
 
 		stats.totalFiles++
 
-		if !hasValidExt(path) {
+		if (!args.AllValidText && !hasValidExt(path)) || (args.AllValidText && !isValidUtf8(path)) {
 			stats.filesNoCode++
 			return nil
 		}
@@ -168,20 +168,28 @@ func checkPath(root string, db hashDB, args *baseArgs) (stats *walkStats) {
 
 		if len(hits) > 0 {
 			stats.filesWithChanges++
+			hasSuspectLines := false
 			logInfo(boldred("\n X " + relPath))
 			for _, idx := range hits {
 				// fmt.Println(string(lines[idx]))
 				if shouldHighlight(lines[idx]) {
+					hasSuspectLines = true
 					logInfo("  ", grey(fmt.Sprintf("%-5d", idx)), alarm(string(lines[idx])))
 					// fmt.Printf("%s %s\n", grey(fmt.Sprintf("%-5d", idx)), alarm(string(lines[idx])))
 				} else if !args.SuspectOnly {
 					logInfo("  ", grey(fmt.Sprintf("%-5d", idx)), string(lines[idx]))
 					// fmt.Printf("%s %s\n", grey(fmt.Sprintf("%-5d", idx)), string(lines[idx]))
 				}
 			}
+			if hasSuspectLines {
+				stats.filesWithSuspectLines++
+			}
 			logInfo()
 		} else {
 			stats.filesWithoutChanges++
+			if args.Verbose {
+				stats.undetectedPaths = append(stats.undetectedPaths, path)
+			}
 			logVerbose(green(" V " + relPath))
 		}
 
@@ -211,9 +219,12 @@ func addPath(root string, db hashDB, args *baseArgs) {
 			return nil
 		}
 
-		if !hasValidExt(path) {
+		if !args.AllValidText && !hasValidExt(path) {
 			logVerbose(grey(" - ", relPath, " (no code)"))
 			return nil
+		} else if !isValidUtf8(path) {
+			logVerbose(grey(" - ", relPath, " (invalid utf8)"))
+			return nil
 		}
 
 		// If relPath has valid ext, add hash of "path:<relPath>" to db
@@ -271,14 +282,23 @@ func main() {
 			logInfo("Found no new code hashes...")
 		}
 	} else {
+		without := "code"
+		if args.AllValidText {
+			without = "text"
+		}
 		for _, path := range args.Path.Path {
 			stats := checkPath(path, db, args)
 			logInfo("\n===============================================================================")
 			logInfo(" Corediff completed scanning", stats.totalFiles, "files in", path)
-			logInfo(" - Files with unrecognized lines   :", boldred(stats.filesWithChanges))
-			logInfo(" - Files with only recognized lines:", green(stats.filesWithoutChanges))
-			logInfo(" - Files with custom code          :", stats.filesCustomCode)
-			logInfo(" - Files without code              :", stats.filesNoCode)
+			logInfo(" - Files with unrecognized lines      :", boldred(stats.filesWithChanges), grey(fmt.Sprintf("%8.2f%%", stats.percentage(stats.filesWithChanges))))
+			logInfo(" - Files with suspect lines           :", warn(stats.filesWithSuspectLines), grey(fmt.Sprintf("%8.2f%%", stats.percentage(stats.filesWithSuspectLines))))
+			logInfo(" - Files with only recognized lines   :", green(stats.filesWithoutChanges), grey(fmt.Sprintf("%8.2f%%", stats.percentage(stats.filesWithoutChanges))))
+			logInfo(" - Files with custom code             :", stats.filesCustomCode, grey(fmt.Sprintf("%8.2f%%", stats.percentage(stats.filesCustomCode))))
+			logInfo(" - Files without", without, "                :", stats.filesNoCode, grey(fmt.Sprintf("%8.2f%%", stats.percentage(stats.filesNoCode))))
+			logVerbose("Undetected paths:")
+			for _, p := range stats.undetectedPaths {
+				logVerbose("  ", p)
+			}
 		}
 	}
 }

go.sum

@@ -5,8 +5,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
@@ -22,27 +20,18 @@ github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

helpers.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/cespare/xxhash/v2"
 )
@@ -36,11 +37,6 @@ func pathExists(p string) bool {
 	return err == nil
 }
 
-func isDir(path string) bool {
-	fi, err := os.Stat(path)
-	return err == nil && fi.IsDir()
-}
-
 func hasValidExt(path string) bool {
 	got := strings.TrimLeft(filepath.Ext(path), ".")
 	for _, want := range scanExts {
@@ -51,6 +47,25 @@ func hasValidExt(path string) bool {
 	return false
 }
 
+func isValidUtf8(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	bytes := make([]byte, 1024*8) // 8 KB
+	if _, err := f.Read(bytes); err != nil {
+		return false
+	}
+
+	valid := utf8.Valid(bytes)
+	if !valid {
+		fmt.Println("Invalid UTF-8:", path)
+	}
+	return valid
+}
+
 func logVerbose(a ...interface{}) {
 	if logLevel >= 3 {
 		fmt.Println(a...)

helpers_test.go

@@ -3,9 +3,6 @@ package main
 import "testing"
 
 func Test_pathIsExcluded(t *testing.T) {
-	type args struct {
-		p string
-	}
 	tests := []struct {
 		arg  string
 		want bool

highlight.go

@@ -1,42 +1,98 @@
 package main
 
 import (
+	"bytes"
 	"regexp"
 )
 
-var highlightPatterns = []string{
-	`fopen\(`,
-	`hex2bin\(`,
-	`die\(`,
-	`exec\(`,
-	`chr\(`,
-	`hexdec\(`,
-	`eval\(`,
-	`system\(`,
-	`shell_exec\(`,
-	`passthru\(`,
-	`popen\(`,
-	`proc_open\(`,
-	`pcntl_exec\(`,
-	`pcntl_fork\(`,
-	`\$_[A-Z]`,
-	`GLOBALS`,
-	`\S"\s*\.\s*"\S`, // " . "
-	`\S'\s*\.\s*'\S`, // ' . '
-	`base64_`,
-	`[a-zA-Z0-9\/\+\=]{40}`, // long base64? string
-	// `@(unlink|include|mysql)`, already more generic one below
-	// `../../..`, // too many fps
-	// `curl_exec,
-	`file_put_contents`,
-	`file_get_contents`,
-	`@[a-z_]{1,16}\(`, // suppressed function call
-	`\$.\(\$.\(`,
-	`call_user_func_array`,
-	`\@\$\w{1,12}\(`, // suppressed dynamic function call
-}
+var (
+	highlightPatternsReg = compileRegexps([]string{
+		// php
+		`\$_[A-Z]`,       // $_GET, $_POST, etc.
+		`\S"\s*\.\s*"\S`, // " . "
+		`\S'\s*\.\s*'\S`, // ' . '
+		`@$?\w{1,16}\(`,  // suppressed function call
+		`\$.\(\$.\(`,     // $x($y(
+		`\@\$\w{1,12}\(`, // suppressed dynamic function call
+		`\/\*\s*\w+\s*\*\/.+\/\*\s*\w+\s*\*\/[^\s]+`,                       // comment obfuscation
+		`include\s{1,10}["'\x60](\w|\/)+\.(png|jpeg|svg|jpg|webp)["'\x60]`, // include php as image
+
+		// common
+		`[a-zA-Z0-9\/\+\=]{25,}`, // long base64 string
+		`(\\x[A-Z0-9]{2}){15,}`,  // long hex string
+		`(_0x\w{4,8}.+){4,}`,     // multiple obfuscated variables
+	})
+	highlightPatternsLit = [][]byte{
+		// php
+		[]byte(`system(`),
+		[]byte(`fopen(`),
+		[]byte(`hex2bin(`),
+		[]byte(`die(`),
+		[]byte(`chr(`),
+		[]byte(`hexdec(`),
+
+		[]byte(`exec`),
+		[]byte(`shell_exec`),
+		[]byte(`passthru`),
+		[]byte(`popen`),
+		[]byte(`system(`),
+		[]byte(`proc_open`),
+		[]byte(`pcntl_exec`),
+		[]byte(`pcntl_fork`),
+		[]byte(`escapeshellcmd`),
+		[]byte(`preg_replace`),
+		[]byte(`create_function`),
+		[]byte(`call_user_func_array`),
+
+		[]byte(`strrev`),
+		[]byte(`str_rot13`),
+		[]byte(`htmlspecialchars_decode`),
+
+		[]byte(`file_get_contents`),
+		[]byte(`file_put_contents`),
+		[]byte(`fwrite`),
+		[]byte(`fread`),
+		[]byte(`fgetc`),
+		[]byte(`fgets`),
+		[]byte(`fscanf`),
+		[]byte(`fgetss`),
+		[]byte(`fpassthru`),
+		[]byte(`readfile`),
+
+		[]byte(`gzuncompress`),
+		[]byte(`gzinflate`),
+		[]byte(`gzdecode`),
+		[]byte(`readgzfile`),
+		[]byte(`gzwrite`),
+		[]byte(`gzfile`),
+
+		[]byte(`umask(`),
+		[]byte(`chmod(`),
+		[]byte(`chown(`),
+		[]byte(`chgrp(`),
+		[]byte(`unlink(`),
+		[]byte(`rmdir(`),
+		[]byte(`mkdir(`),
+		[]byte(`stream_get_meta_data`),
+
+		[]byte(`GLOBALS`),
 
-var highlightPatternsRe = compileRegexps(highlightPatterns)
+		[]byte(`$obirninja`),
+		[]byte(`$pass`),
+		[]byte(`<?php @'$`),
+
+		// js
+		[]byte(`atob`),
+		[]byte(`btoa`),
+		[]byte(`String.fromCharCode(`),
+		[]byte(`jQuery.getScript(`),
+
+		// common
+		[]byte(`../../../../`),
+		[]byte(`base64`),
+		[]byte(`eval`),
+	}
+)
 
 func compileRegexps(patterns []string) []*regexp.Regexp {
 	rxs := make([]*regexp.Regexp, len(patterns))
@@ -47,10 +103,16 @@ func compileRegexps(patterns []string) []*regexp.Regexp {
 }
 
 func shouldHighlight(b []byte) bool {
-	for _, rx := range highlightPatternsRe {
+	for _, p := range highlightPatternsLit {
+		if bytes.Contains(b, p) {
+			return true
+		}
+	}
+	for _, rx := range highlightPatternsReg {
 		if rx.Match(b) {
 			return true
 		}
 	}
+
 	return false
 }

needle_test.go

@@ -15,7 +15,7 @@ func Test_Needle(t *testing.T) {
 	db := loadDB(dbpath)
 	fmt.Println("Loaded entries:", len(db))
 
-	for k, _ := range db {
+	for k := range db {
 		fmt.Println("first entry", k)
 		break
 	}