enhance: remove verb from the generic bf http filter (#1200)

* enhance: remove verb from the generic bf http filter * enhance: fix caddy test since now it picks up the second test subject * enhance: fix caddy test case again and add a negative test suite to otehrs * enhance: add a simply traefik test case * enhance: fix dumps * enhance: fix dumps part 2 * enhance: update index * enhance: stupid class
crowdsecurity · Dec 20, 2024 · cda30eb · cda30eb
1 parent 59b6e7f
commit cda30eb
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 4 deletions.
diff --git a/.index.json b/.index.json
@@ -14373,7 +14373,7 @@
   },
   "crowdsecurity/http-generic-bf": {
    "path": "scenarios/crowdsecurity/http-generic-bf.yaml",
-   "version": "0.6",
+   "version": "0.7",
    "versions": {
     "0.1": {
      "digest": "aaaf0209fe77be79d8d61a50e73e5da6807e8f13eb7d9832e705553770f6d376",
@@ -14398,10 +14398,14 @@
     "0.6": {
      "digest": "2154028ae52c65753b6d7391cfb726041818fd0d443628598ac83f6e3732be53",
      "deprecated": false
+    },
+    "0.7": {
+     "digest": "8f41d98915ac5021d900e8f5b6287d7b9adb16da27b2795d0fc3266790d9f079",
+     "deprecated": false
     }
    },
    "long_description": "QWxlcnQgd2hlbiBhIHNpbmdsZSBJUCB0aGF0IHRyeSB0byBicnV0ZWZvcmNlIGh0dHAgYmFzaWMgYXV0aC4KCkxlYWtzcGVlZCBvZiAxMHMsIGNhcGFjaXR5IG9mIDUuCg==",
-   "content": "IyA0MDQgc2Nhbgp0eXBlOiBsZWFreQojZGVidWc6IHRydWUKbmFtZTogY3Jvd2RzZWN1cml0eS9odHRwLWdlbmVyaWMtYmYKZGVzY3JpcHRpb246ICJEZXRlY3QgZ2VuZXJpYyBodHRwIGJydXRlIGZvcmNlIgpmaWx0ZXI6ICJldnQuTWV0YS5zZXJ2aWNlID09ICdodHRwJyAmJiBldnQuTWV0YS5zdWJfdHlwZSA9PSAnYXV0aF9mYWlsJyIKZ3JvdXBieTogZXZ0Lk1ldGEuc291cmNlX2lwCmNhcGFjaXR5OiA1CmxlYWtzcGVlZDogIjEwcyIKYmxhY2tob2xlOiAxbQpsYWJlbHM6CiAgY29uZmlkZW5jZTogMwogIHNwb29mYWJsZTogMAogIGNsYXNzaWZpY2F0aW9uOgogICAgLSBhdHRhY2suVDExMTAKICBiZWhhdmlvcjogImh0dHA6YnJ1dGVmb3JjZSIKICBsYWJlbDogIkhUVFAgQnJ1dGVmb3JjZSIKICBzZXJ2aWNlOiBodHRwCiAgcmVtZWRpYXRpb246IHRydWUKLS0tCiMgR2VuZXJpYyA0MDEgQXV0aG9yaXphdGlvbiBFcnJvcnMKdHlwZTogbGVha3kKI2RlYnVnOiB0cnVlCm5hbWU6IExlUHJlc2lkZW50ZS9odHRwLWdlbmVyaWMtNDAxLWJmCmRlc2NyaXB0aW9uOiAiRGV0ZWN0IGdlbmVyaWMgNDAxIEF1dGhvcml6YXRpb24gZXJyb3IgYnJ1dGUgZm9yY2UiCmZpbHRlcjogImV2dC5NZXRhLmxvZ190eXBlID09ICdodHRwX2FjY2Vzcy1sb2cnICYmIGV2dC5QYXJzZWQudmVyYiA9PSAnUE9TVCcgJiYgZXZ0Lk1ldGEuaHR0cF9zdGF0dXMgPT0gJzQwMSciCmdyb3VwYnk6IGV2dC5NZXRhLnNvdXJjZV9pcApjYXBhY2l0eTogNQpsZWFrc3BlZWQ6ICIxMHMiCmJsYWNraG9sZTogMW0KbGFiZWxzOgogIGNvbmZpZGVuY2U6IDMKICBzcG9vZmFibGU6IDAKICBjbGFzc2lmaWNhdGlvbjoKICAgIC0gYXR0YWNrLlQxMTEwCiAgYmVoYXZpb3I6ICJodHRwOmJydXRlZm9yY2UiCiAgbGFiZWw6ICJIVFRQIEJydXRlZm9yY2UiCiAgc2VydmljZTogaHR0cAogIHJlbWVkaWF0aW9uOiB0cnVlCi0tLQojIEdlbmVyaWMgNDAzIEZvcmJpZGRlbiAoQXV0aG9yaXphdGlvbikgRXJyb3JzCnR5cGU6IGxlYWt5CiNkZWJ1ZzogdHJ1ZQpuYW1lOiBMZVByZXNpZGVudGUvaHR0cC1nZW5lcmljLTQwMy1iZgpkZXNjcmlwdGlvbjogIkRldGVjdCBnZW5lcmljIDQwMyBGb3JiaWRkZW4gKEF1dGhvcml6YXRpb24pIGVycm9yIGJydXRlIGZvcmNlIgpmaWx0ZXI6ICJldnQuTWV0YS5sb2dfdHlwZSA9PSAnaHR0cF9hY2Nlc3MtbG9nJyAmJiBldnQuUGFyc2VkLnZlcmIgPT0gJ1BPU1QnICYmIGV2dC5NZXRhLmh0dHBfc3RhdHVzID09ICc0MDMnIgpncm91cGJ5OiBldnQuTWV0YS5zb3VyY2VfaXAKY2FwYWNpdHk6IDUKbGVha3NwZWVkOiAiMTBzIgpibGFja2hvbGU6IDFtCmxhYmVsczoKICBjb25maWRlbmNlOiAzCiAgc3Bvb2ZhYmxlOiAwCiAgY2xhc3NpZmljYXRpb246CiAgICAtIGF0dGFjay5UMTExMAogIGJlaGF2aW9yOiAiaHR0cDpicnV0ZWZvcmNlIgogIGxhYmVsOiAiSFRUUCBCcnV0ZWZvcmNlIgogIHNlcnZpY2U6IGh0dHAKICByZW1lZGlhdGlvbjogdHJ1ZQo=",
+   "content": "IyA0MDQgc2Nhbgp0eXBlOiBsZWFreQojZGVidWc6IHRydWUKbmFtZTogY3Jvd2RzZWN1cml0eS9odHRwLWdlbmVyaWMtYmYKZGVzY3JpcHRpb246ICJEZXRlY3QgZ2VuZXJpYyBodHRwIGJydXRlIGZvcmNlIgpmaWx0ZXI6ICJldnQuTWV0YS5zZXJ2aWNlID09ICdodHRwJyAmJiBldnQuTWV0YS5zdWJfdHlwZSA9PSAnYXV0aF9mYWlsJyIKZ3JvdXBieTogZXZ0Lk1ldGEuc291cmNlX2lwCmNhcGFjaXR5OiA1CmxlYWtzcGVlZDogIjEwcyIKYmxhY2tob2xlOiAxbQpsYWJlbHM6CiAgY29uZmlkZW5jZTogMwogIHNwb29mYWJsZTogMAogIGNsYXNzaWZpY2F0aW9uOgogICAgLSBhdHRhY2suVDExMTAKICBiZWhhdmlvcjogImh0dHA6YnJ1dGVmb3JjZSIKICBsYWJlbDogIkhUVFAgQnJ1dGVmb3JjZSIKICBzZXJ2aWNlOiBodHRwCiAgcmVtZWRpYXRpb246IHRydWUKLS0tCiMgR2VuZXJpYyA0MDEgQXV0aG9yaXphdGlvbiBFcnJvcnMKdHlwZTogbGVha3kKI2RlYnVnOiB0cnVlCm5hbWU6IExlUHJlc2lkZW50ZS9odHRwLWdlbmVyaWMtNDAxLWJmCmRlc2NyaXB0aW9uOiAiRGV0ZWN0IGdlbmVyaWMgNDAxIEF1dGhvcml6YXRpb24gZXJyb3IgYnJ1dGUgZm9yY2UiCmZpbHRlcjogImV2dC5NZXRhLmxvZ190eXBlID09ICdodHRwX2FjY2Vzcy1sb2cnICYmIGV2dC5NZXRhLmh0dHBfc3RhdHVzID09ICc0MDEnICYmIGV2dC5NZXRhLnN1Yl90eXBlICE9ICdhdXRoX2ZhaWwnIgpncm91cGJ5OiBldnQuTWV0YS5zb3VyY2VfaXAKY2FwYWNpdHk6IDUKbGVha3NwZWVkOiAiMTBzIgpibGFja2hvbGU6IDFtCmxhYmVsczoKICBjb25maWRlbmNlOiAzCiAgc3Bvb2ZhYmxlOiAwCiAgY2xhc3NpZmljYXRpb246CiAgICAtIGF0dGFjay5UMTExMAogIGJlaGF2aW9yOiAiaHR0cDpicnV0ZWZvcmNlIgogIGxhYmVsOiAiSFRUUCBCcnV0ZWZvcmNlIgogIHNlcnZpY2U6IGh0dHAKICByZW1lZGlhdGlvbjogdHJ1ZQotLS0KIyBHZW5lcmljIDQwMyBGb3JiaWRkZW4gKEF1dGhvcml6YXRpb24pIEVycm9ycwp0eXBlOiBsZWFreQojZGVidWc6IHRydWUKbmFtZTogTGVQcmVzaWRlbnRlL2h0dHAtZ2VuZXJpYy00MDMtYmYKZGVzY3JpcHRpb246ICJEZXRlY3QgZ2VuZXJpYyA0MDMgRm9yYmlkZGVuIChBdXRob3JpemF0aW9uKSBlcnJvciBicnV0ZSBmb3JjZSIKZmlsdGVyOiAiZXZ0Lk1ldGEubG9nX3R5cGUgPT0gJ2h0dHBfYWNjZXNzLWxvZycgJiYgZXZ0Lk1ldGEuaHR0cF9zdGF0dXMgPT0gJzQwMycgJiYgZXZ0Lk1ldGEuc3ViX3R5cGUgIT0gJ2F1dGhfZmFpbCciCmdyb3VwYnk6IGV2dC5NZXRhLnNvdXJjZV9pcApjYXBhY2l0eTogNQpsZWFrc3BlZWQ6ICIxMHMiCmJsYWNraG9sZTogMW0KbGFiZWxzOgogIGNvbmZpZGVuY2U6IDMKICBzcG9vZmFibGU6IDAKICBjbGFzc2lmaWNhdGlvbjoKICAgIC0gYXR0YWNrLlQxMTEwCiAgYmVoYXZpb3I6ICJodHRwOmJydXRlZm9yY2UiCiAgbGFiZWw6ICJIVFRQIEJydXRlZm9yY2UiCiAgc2VydmljZTogaHR0cAogIHJlbWVkaWF0aW9uOiB0cnVlCgo=",
    "description": "Detect generic http brute force",
    "author": "crowdsecurity",
    "labels": {

diff --git a/.tests/traefik_http_generic_bf/config.yaml b/.tests/traefik_http_generic_bf/config.yaml
@@ -0,0 +1,13 @@
+parsers:
+- ./parsers/s01-parse/crowdsecurity/traefik-logs.yaml
+- crowdsecurity/syslog-logs
+- crowdsecurity/dateparse-enrich
+- crowdsecurity/http-logs
+scenarios:
+- crowdsecurity/http-generic-bf
+postoverflows:
+- ""
+log_file: traefik_generic-bf.log
+log_type: traefik
+labels: {}
+ignore_parsers: true 
diff --git a/.tests/traefik_http_generic_bf/parser.assert b/.tests/traefik_http_generic_bf/parser.assert
diff --git a/.tests/traefik_http_generic_bf/scenario.assert b/.tests/traefik_http_generic_bf/scenario.assert
@@ -0,0 +1,87 @@
+len(results) == 1
+"192.168.1.1" in results[0].Overflow.GetSources()
+results[0].Overflow.Sources["192.168.1.1"].IP == "192.168.1.1"
+results[0].Overflow.Sources["192.168.1.1"].Range == ""
+results[0].Overflow.Sources["192.168.1.1"].GetScope() == "Ip"
+results[0].Overflow.Sources["192.168.1.1"].GetValue() == "192.168.1.1"
+results[0].Overflow.Alert.Events[0].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[0].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[0].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[0].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[0].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[0].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[0].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[0].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[0].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[0].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[0].GetMeta("timestamp") == "2024-12-17T15:59:28Z"
+results[0].Overflow.Alert.Events[0].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[0].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.Events[1].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[1].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[1].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[1].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[1].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[1].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[1].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[1].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[1].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[1].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[1].GetMeta("timestamp") == "2024-12-17T15:59:29Z"
+results[0].Overflow.Alert.Events[1].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[1].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.Events[2].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[2].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[2].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[2].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[2].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[2].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[2].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[2].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[2].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[2].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[2].GetMeta("timestamp") == "2024-12-17T15:59:30Z"
+results[0].Overflow.Alert.Events[2].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[2].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.Events[3].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[3].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[3].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[3].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[3].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[3].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[3].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[3].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[3].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[3].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[3].GetMeta("timestamp") == "2024-12-17T15:59:31Z"
+results[0].Overflow.Alert.Events[3].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[3].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.Events[4].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[4].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[4].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[4].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[4].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[4].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[4].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[4].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[4].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[4].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[4].GetMeta("timestamp") == "2024-12-17T15:59:32Z"
+results[0].Overflow.Alert.Events[4].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[4].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.Events[5].GetMeta("datasource_path") == "traefik_generic-bf.log"
+results[0].Overflow.Alert.Events[5].GetMeta("datasource_type") == "file"
+results[0].Overflow.Alert.Events[5].GetMeta("http_args_len") == "0"
+results[0].Overflow.Alert.Events[5].GetMeta("http_path") == "/"
+results[0].Overflow.Alert.Events[5].GetMeta("http_status") == "401"
+results[0].Overflow.Alert.Events[5].GetMeta("http_user_agent") == "-"
+results[0].Overflow.Alert.Events[5].GetMeta("http_verb") == "GET"
+results[0].Overflow.Alert.Events[5].GetMeta("log_type") == "http_access-log"
+results[0].Overflow.Alert.Events[5].GetMeta("service") == "http"
+results[0].Overflow.Alert.Events[5].GetMeta("source_ip") == "192.168.1.1"
+results[0].Overflow.Alert.Events[5].GetMeta("timestamp") == "2024-12-17T15:59:33Z"
+results[0].Overflow.Alert.Events[5].GetMeta("traefik_router_name") == "whoami-new@docker"
+results[0].Overflow.Alert.Events[5].GetMeta("user") == "fgjh"
+results[0].Overflow.Alert.GetScenario() == "LePresidente/http-generic-401-bf"
+results[0].Overflow.Alert.Remediation == true
+results[0].Overflow.Alert.GetEventsCount() == 6
diff --git a/.tests/traefik_http_generic_bf/traefik_generic-bf.log b/.tests/traefik_http_generic_bf/traefik_generic-bf.log
@@ -0,0 +1,7 @@
+192.168.1.1 - fgjh [17/Dec/2024:15:59:28 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:29 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:30 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:31 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:32 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:33 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
+192.168.1.1 - fgjh [17/Dec/2024:15:59:34 +0000] "GET / HTTP/2.0" 401 17 "-" "-" 108709 "whoami-new@docker" "-" 1ms
diff --git a/scenarios/crowdsecurity/http-generic-bf.yaml b/scenarios/crowdsecurity/http-generic-bf.yaml
@@ -23,7 +23,7 @@ type: leaky
 #debug: true
 name: LePresidente/http-generic-401-bf
 description: "Detect generic 401 Authorization error brute force"
-filter: "evt.Meta.log_type == 'http_access-log' && evt.Parsed.verb == 'POST' && evt.Meta.http_status == '401'"
+filter: "evt.Meta.log_type == 'http_access-log' && evt.Meta.http_status == '401' && evt.Meta.sub_type != 'auth_fail'"
 groupby: evt.Meta.source_ip
 capacity: 5
 leakspeed: "10s"
@@ -43,7 +43,7 @@ type: leaky
 #debug: true
 name: LePresidente/http-generic-403-bf
 description: "Detect generic 403 Forbidden (Authorization) error brute force"
-filter: "evt.Meta.log_type == 'http_access-log' && evt.Parsed.verb == 'POST' && evt.Meta.http_status == '403'"
+filter: "evt.Meta.log_type == 'http_access-log' && evt.Meta.http_status == '403' && evt.Meta.sub_type != 'auth_fail'"
 groupby: evt.Meta.source_ip
 capacity: 5
 leakspeed: "10s"
@@ -57,3 +57,4 @@ labels:
   label: "HTTP Bruteforce"
   service: http
   remediation: true
+