From 2adda59258187140770daca9bb71fdedb342fdd7 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Sun, 25 Aug 2024 20:19:49 +0200 Subject: [PATCH 01/12] feat(stdlib): Initial implementation of parse_auditd function --- Cargo.lock | 30 +++++-- Cargo.toml | 2 + src/stdlib/mod.rs | 3 + src/stdlib/parse_auditd.rs | 179 +++++++++++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 src/stdlib/parse_auditd.rs diff --git a/Cargo.lock b/Cargo.lock index eeb908efe..ce3b180b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1499,9 +1499,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" @@ -1515,6 +1515,19 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "linux-audit-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a4056b37f64fbc83d1bc0ffdabd77a9238fadf46677cbc7f9e373e8389f9865" +dependencies = [ + "indexmap 2.4.0", + "lazy_static", + "nom", + "thiserror", + "tinyvec", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2749,18 +2762,18 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", @@ -3152,9 +3165,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -3502,6 +3515,7 @@ dependencies = [ "itertools 0.13.0", "lalrpop", "lalrpop-util", + "linux-audit-parser", "md-5", "mlua", "nom", diff --git a/Cargo.toml b/Cargo.toml index 97464c201..58e01d76d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ stdlib = [ "dep:idna", "dep:indexmap", "dep:influxdb-line-protocol", + "dep:linux-audit-parser", "dep:md-5", "dep:nom", "dep:ofb", @@ -146,6 +147,7 @@ influxdb-line-protocol = { version = "2.0.0", optional = true } indoc = {version = "2", optional = true } itertools = { version = "0.13", default-features = false, optional = true } lalrpop-util = { version = "0.20", optional = true } +linux-audit-parser = { version = "0.1.1", default-features = false, optional = true } mlua = { version = "0.9", default-features = false, features = ["lua54", "send", "vendored"], optional = true} nom = { version = "7", default-features = false, features = ["std"], optional = true } once_cell = { version = "1", default-features = false, features = ["std"], optional = true } diff --git a/src/stdlib/mod.rs b/src/stdlib/mod.rs index bd0b85a25..e6925ad95 100644 --- a/src/stdlib/mod.rs +++ b/src/stdlib/mod.rs @@ -133,6 +133,7 @@ cfg_if::cfg_if! { mod now; mod object; mod parse_apache_log; + mod parse_auditd; mod parse_aws_alb_log; mod parse_aws_cloudwatch_log_subscription_message; mod parse_aws_vpc_flow_log; @@ -312,6 +313,7 @@ cfg_if::cfg_if! { pub use now::Now; pub use object::Object; pub use parse_apache_log::ParseApacheLog; + pub use parse_auditd::ParseAuditd; pub use parse_aws_alb_log::ParseAwsAlbLog; pub use parse_aws_cloudwatch_log_subscription_message::ParseAwsCloudWatchLogSubscriptionMessage; pub use parse_aws_vpc_flow_log::ParseAwsVpcFlowLog; @@ -495,6 +497,7 @@ pub fn all() -> Vec> { Box::new(Now), Box::new(Object), Box::new(ParseApacheLog), + Box::new(ParseAuditd), Box::new(ParseAwsAlbLog), Box::new(ParseAwsCloudWatchLogSubscriptionMessage), Box::new(ParseAwsVpcFlowLog), diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs new file mode 100644 index 000000000..ba4750758 --- /dev/null +++ b/src/stdlib/parse_auditd.rs @@ -0,0 +1,179 @@ +use crate::compiler::prelude::*; +use bytes::BytesMut; +use chrono::DateTime; +use linux_audit_parser::{Number, Value as AuditValue}; + +fn parse_auditd(bytes: Value) -> Resolved { + let bytes = bytes.try_bytes()?; + // check if bytes ends with newline, otherwise append it + let bytes = if bytes.last() == Some(&b'\n') { + bytes + } else { + let mut bytes = BytesMut::from(bytes); + bytes.extend_from_slice(b"\n"); + bytes.freeze() + }; + let parsed = linux_audit_parser::parse(&bytes, false)?; + + let mut log = ObjectMap::new(); + + let timestamp_millis = parsed.id.timestamp as i64; + // Should we use UTC as the timezone? The logs are generated with the system's + // timezone... What is the correct behavior? Maybe the system where vector is running + // have a different timezone than the system that generated the logs... so it is + // not correct to assume that the current system's timezone is the correct one + // (with TimeZone::timestamp_millis_opt) + let Some(timestamp) = DateTime::from_timestamp_millis(timestamp_millis) else { + return Err(Error::TimestampOutOfRange(timestamp_millis).into()); + }; + log.insert("timestamp".into(), Value::from(timestamp)); + + let sequence = parsed.id.sequence; + log.insert("sequence".into(), Value::from(sequence)); + + if let Some(node) = parsed.node { + log.insert("node".into(), Value::from(node)); + } + + let message_type = parsed.ty.to_string(); + log.insert("type".into(), Value::from(message_type)); + + let (enrichment, body): (ObjectMap, ObjectMap) = parsed + .body + .into_iter() + // TODO: improve this auditd crate with a IntoIter implementation for Body and not only + // for &Body, so we can have owned values + .map(|(key, value)| { + let key = KeyString::from(key.to_string()); + // TODO: remove this clone with a new auditd crate version + let value = Value::from(value.clone()); + (key, value) + }) + // partition whether the key is all caps or not + .partition(|(key, _)| key.chars().all(char::is_uppercase)); + + log.insert("body".into(), Value::from(body)); + log.insert("enrichment".into(), Value::from(enrichment)); + + Ok(Value::from(log)) +} + +impl From> for Value { + fn from(value: AuditValue) -> Self { + match value { + AuditValue::Str(buf, _) => Value::from(buf), + AuditValue::Number(num) => Value::from(num), + value => Value::from(format!("TODO {:?}", value)), + } + } +} + +impl From for Value { + fn from(number: Number) -> Self { + match number { + Number::Dec(decimal) => Value::from(decimal), + // TODO: should we store hexadecimals as its integer value or as an hexadecimal string? + // Uppercase hexa or lowercase hex format? + Number::Hex(hex) => Value::from(format!("0x{hex:X}")), + // TODO: should we store octals as its integer value or as an octal string? + Number::Oct(oct) => Value::from(format!("0o{oct:o}")), + } + } +} + +#[derive(Debug, Clone, thiserror::Error)] +enum Error { + #[error("timestamp (in milliseconds) {0} is out of range")] + TimestampOutOfRange(i64), +} + +impl From for ExpressionError { + fn from(error: Error) -> Self { + Self::Error { + message: format!("Error while converting parsed Auditd record to object: {error}"), + labels: vec![], + notes: vec![], + } + } +} + +impl From for ExpressionError { + fn from(error: linux_audit_parser::ParseError) -> Self { + Self::Error { + message: format!("Auditd record parsing error: {error}"), + labels: vec![], + notes: vec![], + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ParseAuditd; + +impl Function for ParseAuditd { + fn identifier(&self) -> &'static str { + "parse_auditd" + } + + fn summary(&self) -> &'static str { + "Parse an auditd record" + } + + fn parameters(&self) -> &'static [Parameter] { + &[Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }] + } + + fn examples(&self) -> &'static [Example] { + // TODO + &[] + } + + fn compile( + &self, + _state: &state::TypeState, + _ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let value = arguments.required("value"); + + Ok(ParseAuditdFn { value }.as_expr()) + } +} + +#[derive(Clone, Debug)] +struct ParseAuditdFn { + value: Box, +} + +impl FunctionExpression for ParseAuditdFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + let value = self.value.resolve(ctx)?; + + parse_auditd(value) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + type_def() + } +} + +fn type_def() -> TypeDef { + // TODO: improve typedef + TypeDef::object(Collection::any()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_auditd() { + let line = r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=successAUID="jorge" UID="root""#; + let value = Value::from(line); + println!("{}", parse_auditd(value).expect("expected ok ")); + } +} From 091d59b3bab7bba6307b580911f7b46b70a05146 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Tue, 27 Aug 2024 22:35:06 +0200 Subject: [PATCH 02/12] feat(stdlib): Initial implementation of auditd functiopn --- src/stdlib/parse_auditd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index ba4750758..49cfaf6b1 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -6,6 +6,7 @@ use linux_audit_parser::{Number, Value as AuditValue}; fn parse_auditd(bytes: Value) -> Resolved { let bytes = bytes.try_bytes()?; // check if bytes ends with newline, otherwise append it + // TODO: make the parser accept bytes without newline in the linux_audit_parser crate let bytes = if bytes.last() == Some(&b'\n') { bytes } else { From 987fcd27b3d57fae090d7ab1897f32e8d8bdb32f Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 17:39:03 +0200 Subject: [PATCH 03/12] feat(stdlib): Initial implementation of auditd functiopn --- src/stdlib/parse_auditd.rs | 70 +++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 49cfaf6b1..f1ae78b08 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -18,12 +18,15 @@ fn parse_auditd(bytes: Value) -> Resolved { let mut log = ObjectMap::new(); - let timestamp_millis = parsed.id.timestamp as i64; + let timestamp_millis = i64::try_from(parsed.id.timestamp) + .map_err(|_| Error::TimestampOverflow(parsed.id.timestamp))?; // Should we use UTC as the timezone? The logs are generated with the system's // timezone... What is the correct behavior? Maybe the system where vector is running // have a different timezone than the system that generated the logs... so it is // not correct to assume that the current system's timezone is the correct one // (with TimeZone::timestamp_millis_opt) + // TODO: we should document that the timestamp is parsed into UTC timezone. + // TODO: Maybe we should accept in a parameter a custom timezone to parse the timestamp? let Some(timestamp) = DateTime::from_timestamp_millis(timestamp_millis) else { return Err(Error::TimestampOutOfRange(timestamp_millis).into()); }; @@ -46,26 +49,59 @@ fn parse_auditd(bytes: Value) -> Resolved { // for &Body, so we can have owned values .map(|(key, value)| { let key = KeyString::from(key.to_string()); - // TODO: remove this clone with a new auditd crate version - let value = Value::from(value.clone()); - (key, value) + // TODO: remove this clone with a new auditd crate version (to-be-contributed yet) + let value = Value::try_from(value.clone()); + value.map(|value| (key, value)) }) + .collect::, _>>()? // partition whether the key is all caps or not + .into_iter() .partition(|(key, _)| key.chars().all(char::is_uppercase)); log.insert("body".into(), Value::from(body)); + // TODO: do not insert enrichment if it is empty. Create a test for this log.insert("enrichment".into(), Value::from(enrichment)); Ok(Value::from(log)) } -impl From> for Value { - fn from(value: AuditValue) -> Self { - match value { - AuditValue::Str(buf, _) => Value::from(buf), +impl<'a> TryFrom> for Value { + type Error = Error<'a>; + fn try_from(value: AuditValue<'a>) -> Result { + let result = match value { + AuditValue::Empty => Value::Null, + AuditValue::Str(string, _) => Value::from(string), AuditValue::Number(num) => Value::from(num), - value => Value::from(format!("TODO {:?}", value)), - } + AuditValue::List(values) => Value::from( + values + .into_iter() + .map(Value::try_from) + .collect::, _>>()?, + ), + AuditValue::Owned(string) => Value::from(string), + // Currently, AuditValue::Map is not generated from the parser (https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/value.rs#L70) + // but I think we should use that data type to represent the key-value pairs of the internal (nested) msg field + // as depicted here: https://github.com/vectordotdev/vrl/issues/66 + // For example, in the following linux-audit-parser test https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/test.rs#L168 + // we see that the msg field is parsed as a string, although it could be key-value pairs + AuditValue::Map(entries) => Value::from( + entries + .into_iter() + .map(|(key, value)| { + Value::try_from(value).map(|value| (key.to_string().into(), value)) + }) + .collect::>()?, + ), + // There is a few values that `linux-audit-parser` does not return in its parsing + // https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/value.rs#L72 + // We do not plan to support those values, as they are only produced by [laurel](https://github.com/threathunters-io/laurel) + // Maybe we should contribute to `linux-audit-parser` to remove the values the parser does not produce, as it + // does not have sense in this context. + // If those variants are removed, we can remove the error handling of this cases are simplify the code. + // Otherwise, we could simply panic in this case + unsupported_value => return Err(Error::UnsupportedValue(unsupported_value)), + }; + Ok(result) } } @@ -74,7 +110,7 @@ impl From for Value { match number { Number::Dec(decimal) => Value::from(decimal), // TODO: should we store hexadecimals as its integer value or as an hexadecimal string? - // Uppercase hexa or lowercase hex format? + // TODO: Uppercase hexa or lowercase hex format? Number::Hex(hex) => Value::from(format!("0x{hex:X}")), // TODO: should we store octals as its integer value or as an octal string? Number::Oct(oct) => Value::from(format!("0o{oct:o}")), @@ -83,12 +119,16 @@ impl From for Value { } #[derive(Debug, Clone, thiserror::Error)] -enum Error { +pub enum Error<'a> { #[error("timestamp (in milliseconds) {0} is out of range")] TimestampOutOfRange(i64), + #[error("timestamp {0} overflow while converting from u64 to i64")] + TimestampOverflow(u64), + #[error("unsupported auditd value: {0:?}")] + UnsupportedValue(AuditValue<'a>), } -impl From for ExpressionError { +impl From> for ExpressionError { fn from(error: Error) -> Self { Self::Error { message: format!("Error while converting parsed Auditd record to object: {error}"), @@ -176,5 +216,9 @@ mod tests { let line = r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=successAUID="jorge" UID="root""#; let value = Value::from(line); println!("{}", parse_auditd(value).expect("expected ok ")); + + let other_line = r#"type=SYSCALL msg=audit(1522927552.749:917): arch=c000003e syscall=2 success=yes exit=3 a0=7ffe2ce05793 a1=0 a2=1fffffffffff0000 a3=7ffe2ce043a0 items=1 ppid=2906 pid=4668 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts4 ses=1 comm="cat" exe="/bin/cat" key="passwd""#; + let value = Value::from(other_line); + println!("{}", parse_auditd(value).expect("expected ok ")); } } From 42e44f6a279c66fb095c0672771cb770ece516cc Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 19:33:40 +0200 Subject: [PATCH 04/12] feat(stdlib): Initial implementation of auditd functiopn --- src/stdlib/parse_auditd.rs | 267 ++++++++++++++++++++++++++++++++++--- 1 file changed, 245 insertions(+), 22 deletions(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index f1ae78b08..d5fc427ce 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -1,4 +1,6 @@ -use crate::compiler::prelude::*; +use std::collections::BTreeMap; + +use crate::{btreemap, compiler::prelude::*}; use bytes::BytesMut; use chrono::DateTime; use linux_audit_parser::{Number, Value as AuditValue}; @@ -6,7 +8,7 @@ use linux_audit_parser::{Number, Value as AuditValue}; fn parse_auditd(bytes: Value) -> Resolved { let bytes = bytes.try_bytes()?; // check if bytes ends with newline, otherwise append it - // TODO: make the parser accept bytes without newline in the linux_audit_parser crate + // TODO: make the parser accept bytes without newline in the linux_audit_parser crate (to-be-contributed) let bytes = if bytes.last() == Some(&b'\n') { bytes } else { @@ -36,7 +38,7 @@ fn parse_auditd(bytes: Value) -> Resolved { log.insert("sequence".into(), Value::from(sequence)); if let Some(node) = parsed.node { - log.insert("node".into(), Value::from(node)); + log.insert("node".into(), Value::from(Bytes::from(node))); } let message_type = parsed.ty.to_string(); @@ -46,7 +48,8 @@ fn parse_auditd(bytes: Value) -> Resolved { .body .into_iter() // TODO: improve this auditd crate with a IntoIter implementation for Body and not only - // for &Body, so we can have owned values + // for &Body, so we can have owned values. currently, the `into_iter` `Body` implementation + // relies on Vec::iter instead of Vec::into_iter, which forces us to clone the value in L56 .map(|(key, value)| { let key = KeyString::from(key.to_string()); // TODO: remove this clone with a new auditd crate version (to-be-contributed yet) @@ -54,13 +57,16 @@ fn parse_auditd(bytes: Value) -> Resolved { value.map(|value| (key, value)) }) .collect::, _>>()? - // partition whether the key is all caps or not .into_iter() + // Keys with only uppercase characters are considered enrichments of its lowercase counterpart + // https://github.com/linux-audit/audit-documentation/wiki/SPEC-Audit-Event-Enrichment .partition(|(key, _)| key.chars().all(char::is_uppercase)); log.insert("body".into(), Value::from(body)); - // TODO: do not insert enrichment if it is empty. Create a test for this - log.insert("enrichment".into(), Value::from(enrichment)); + + if !enrichment.is_empty() { + log.insert("enrichment".into(), Value::from(enrichment)); + } Ok(Value::from(log)) } @@ -78,7 +84,7 @@ impl<'a> TryFrom> for Value { .map(Value::try_from) .collect::, _>>()?, ), - AuditValue::Owned(string) => Value::from(string), + AuditValue::Owned(string) => Value::from(Bytes::from(string)), // Currently, AuditValue::Map is not generated from the parser (https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/value.rs#L70) // but I think we should use that data type to represent the key-value pairs of the internal (nested) msg field // as depicted here: https://github.com/vectordotdev/vrl/issues/66 @@ -111,7 +117,7 @@ impl From for Value { Number::Dec(decimal) => Value::from(decimal), // TODO: should we store hexadecimals as its integer value or as an hexadecimal string? // TODO: Uppercase hexa or lowercase hex format? - Number::Hex(hex) => Value::from(format!("0x{hex:X}")), + Number::Hex(hex) => Value::from(format!("0x{hex:x}")), // TODO: should we store octals as its integer value or as an octal string? Number::Oct(oct) => Value::from(format!("0o{oct:o}")), } @@ -157,7 +163,7 @@ impl Function for ParseAuditd { } fn summary(&self) -> &'static str { - "Parse an auditd record" + "Parse an auditd log record" } fn parameters(&self) -> &'static [Parameter] { @@ -170,7 +176,32 @@ impl Function for ParseAuditd { fn examples(&self) -> &'static [Example] { // TODO - &[] + &[Example { + title: "parse auditd log", + source: "parse_auditd(\"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success\0x1dAUID=\\\"vrl\\\" UID=\\\"root\\\"\")", + result: Ok(indoc! {r#" + { + "body": { + "auid": 1000, + "format": "enriched", + "kernel": "6.10.4-arch2-1", + "op": "start", + "pid": 1240242, + "res": "success", + "ses": 2, + "uid": 0, + "ver": "4.0.2" + }, + "enrichment": { + "AUID": "vrl", + "UID": "root" + }, + "sequence": 6439, + "timestamp": "2024-08-23T14:27:54.618Z", + "type": "DAEMON_START" + } + "#}), + }] } fn compile( @@ -202,23 +233,215 @@ impl FunctionExpression for ParseAuditdFn { } } +fn body_kind() -> Kind { + Kind::object(Collection::any()) +} + +fn enrichment_kind() -> Kind { + Kind::object(Collection::any()) | Kind::null() +} + +fn inner_kind() -> BTreeMap { + btreemap! { + "body" => body_kind(), + "enrichment" => enrichment_kind(), + "sequence" => Kind::integer(), + "timestamp" => Kind::timestamp(), + "type" => Kind::bytes(), + "node" => Kind::bytes() | Kind::null() + } +} + fn type_def() -> TypeDef { - // TODO: improve typedef - TypeDef::object(Collection::any()) + TypeDef::object(inner_kind()) } #[cfg(test)] mod tests { use super::*; + const ENRICHMENT_SEPARATOR: char = 0x1d as char; - #[test] - fn test_parse_auditd() { - let line = r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=successAUID="jorge" UID="root""#; - let value = Value::from(line); - println!("{}", parse_auditd(value).expect("expected ok ")); + // #[test] + // fn test_parse_auditd() { + // let line = r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=successAUID="jorge" UID="root""#; + // let value = Value::from(line); + // println!("{}", parse_auditd(value).expect("expected ok ")); - let other_line = r#"type=SYSCALL msg=audit(1522927552.749:917): arch=c000003e syscall=2 success=yes exit=3 a0=7ffe2ce05793 a1=0 a2=1fffffffffff0000 a3=7ffe2ce043a0 items=1 ppid=2906 pid=4668 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts4 ses=1 comm="cat" exe="/bin/cat" key="passwd""#; - let value = Value::from(other_line); - println!("{}", parse_auditd(value).expect("expected ok ")); - } + // let other_line = r#"type=SYSCALL msg=audit(1522927552.749:917): arch=c000003e syscall=2 success=yes exit=3 a0=7ffe2ce05793 a1=0 a2=1fffffffffff0000 a3=7ffe2ce043a0 items=1 ppid=2906 pid=4668 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts4 ses=1 comm="cat" exe="/bin/cat" key="passwd""#; + // let value = Value::from(other_line); + // println!("{}", parse_auditd(value).expect("expected ok ")); + // } + + test_function![ + parse_auditd => ParseAuditd; + + daemon_start { + args: func_args![value: format!(r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success{}AUID="vrl" UID="root""#, + ENRICHMENT_SEPARATOR)], + want: Ok(btreemap! { + "type" => "DAEMON_START", + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, + "body" => btreemap! { + "op" => "start", + "ver" => "4.0.2", + "format" => "enriched", + "kernel" => "6.10.4-arch2-1", + "auid" => 1000, + "pid" => 1_240_242, + "uid" => 0, + "ses" => 2, + "res" => "success" + }, + "enrichment" => btreemap! { + "AUID" => "vrl", + "UID" => "root" + } + }), + tdef: type_def(), + } + + daemon_start_with_node { + args: func_args![value: format!(r#"node=vrl-node type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success{}AUID="vrl" UID="root""#, + ENRICHMENT_SEPARATOR)], + want: Ok(btreemap! { + "node" => "vrl-node", + "type" => "DAEMON_START", + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, + "body" => btreemap! { + "op" => "start", + "ver" => "4.0.2", + "format" => "enriched", + "kernel" => "6.10.4-arch2-1", + "auid" => 1000, + "pid" => 1_240_242, + "uid" => 0, + "ses" => 2, + "res" => "success" + }, + "enrichment" => btreemap! { + "AUID" => "vrl", + "UID" => "root" + } + }), + tdef: type_def(), + } + + // TODO: anonymize all those tests cases with random values while keeping actual syntax + // TODO: include this in examples + syscall { + args: func_args![ value: format!(r#"type=SYSCALL msg=audit(1615114232.375:15558): arch=c000003e syscall=59 success=yes exit=0 a0=63b29337fd18 a1=63b293387d58 a2=63b293375640 a3=fffffffffffff000 items=2 ppid=10883 pid=10884 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts1 ses=1 comm="whoami" exe="/usr/bin/whoami" key=(null){}ARCH=x86_64 SYSCALL=execve AUID="user" UID="root" GID="root" EUID="root" SUID="root" FSUID="root" EGID="root" SGID="root" FSGID="root""#, + ENRICHMENT_SEPARATOR) ], + want: Ok(btreemap! { + "type" => "SYSCALL", + "timestamp" => DateTime::from_timestamp_millis(1_615_114_232_375), + "sequence" => 15558, + + "body" => btreemap! { + "arch" => "0xc000003e", + "syscall" => 59, + "success" => "yes", + "exit" => 0, + "a0" => "0x63b29337fd18", + "a1" => "0x63b293387d58", + "a2" => "0x63b293375640", + "a3" => "0xfffffffffffff000", + "items" => 2, + "ppid" => 10_883, + "pid" => 10_884, + "auid" => 1000, + "uid" => 0, + "gid" => 0, + "euid" => 0, + "suid" => 0, + "fsuid" => 0, + "egid" => 0, + "sgid" => 0, + "fsgid" => 0, + "tty" => "pts1", + "ses" => 1, + "comm" => "whoami", + "exe" => "/usr/bin/whoami", + "key" => Value::Null, + }, + "enrichment" => btreemap! { + "ARCH" => "x86_64", + "SYSCALL" => "execve", + "AUID" => "user", + "UID" => "root", + "GID" => "root", + "EUID" => "root", + "SUID" => "root", + "FSUID" => "root", + "EGID" => "root", + "SGID" => "root", + "FSGID" => "root" + } + }), + tdef: type_def(), + } + + // TODO: include this different types (denied Array) in examples + avc_denied { + args: func_args![value: r#"type=AVC msg=audit(1631798689.083:65686): avc: denied { setuid } for pid=15381 comm="laurel" capability=7 scontext=system_u:system_r:auditd_t:s0 tcontext=system_u:system_r:auditd_t:s0 tclass=capability permissive=1"#], + want: Ok(btreemap! { + "type" => "AVC", + "timestamp" => DateTime::from_timestamp_millis(1_631_798_689_083), + "sequence" => 65686, + "body" => btreemap! { + "denied" => vec!["setuid"], + "pid" => 15_381, + "comm" => "laurel", + "capability" => 7, + "scontext" => "system_u:system_r:auditd_t:s0", + "tcontext" => "system_u:system_r:auditd_t:s0", + "tclass" => "capability", + "permissive" => 1 + } + }), + tdef: type_def(), + } + + avc_granted{ + args: func_args![value: r#"type=AVC msg=audit(1631870323.500:7098): avc: granted { setsecparam } for pid=11209 comm="tuned" scontext=system_u:system_r:tuned_t:s0 tcontext=system_u:object_r:security_t:s0 tclass=security"#], + want: Ok(btreemap! { + "type" => "AVC", + "timestamp" => DateTime::from_timestamp_millis(1_631_870_323_500), + "sequence" => 7098, + "body" => btreemap! { + "granted" => vec!["setsecparam"], + "pid" => 11_209, + "comm" => "tuned", + "scontext" => "system_u:system_r:tuned_t:s0", + "tcontext" => "system_u:object_r:security_t:s0", + "tclass" => "security" + } + }), + tdef: type_def(), + } + + user_acct { + args: func_args![value: format!(r#"type=USER_ACCT msg=audit(1724505830.648:19): pid=445523 uid=1000 auid=1000 ses=2 msg='op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="jorge" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success'{}UID="jorge" AUID="jorge""#, ENRICHMENT_SEPARATOR)], + want: Ok(btreemap! { + "type" => "USER_ACCT", + "timestamp" => DateTime::from_timestamp_millis(1_724_505_830_648), + "sequence" => 19, + "body" => btreemap! { + "pid" => 445_523, + "uid" => 1_000, + "auid" => 1_000, + "ses" => 2, + // TODO: this should be parsed into a nested object, but it is lack of implementation + // from the linux-audit-parse crate + "msg" => r#"op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="jorge" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success"#, + }, + "enrichment" => btreemap! { + "UID" => "jorge", + "AUID" => "jorge" + } + }), + tdef: type_def(), + } + ]; } From 890a9c358a3c2ff0f93179a98f19d9071997784d Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 19:33:56 +0200 Subject: [PATCH 05/12] feat(stdlib): Initial implementation of auditd functiopn --- src/stdlib/parse_auditd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index d5fc427ce..43b73f8ef 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -175,7 +175,7 @@ impl Function for ParseAuditd { } fn examples(&self) -> &'static [Example] { - // TODO + // TODO: add more examples based on tests &[Example { title: "parse auditd log", source: "parse_auditd(\"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success\0x1dAUID=\\\"vrl\\\" UID=\\\"root\\\"\")", From c86d20822fd7329fccc8bff4daaca7014ce5a2d8 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 19:34:12 +0200 Subject: [PATCH 06/12] feat(stdlib): Initial implementation of auditd functiopn --- src/stdlib/parse_auditd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 43b73f8ef..27d314b41 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -98,7 +98,7 @@ impl<'a> TryFrom> for Value { }) .collect::>()?, ), - // There is a few values that `linux-audit-parser` does not return in its parsing + // There are a few values that `linux-audit-parser` does not return in its parsing // https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/value.rs#L72 // We do not plan to support those values, as they are only produced by [laurel](https://github.com/threathunters-io/laurel) // Maybe we should contribute to `linux-audit-parser` to remove the values the parser does not produce, as it From 3c515a67a6294a6aa1d857e1b4d411d07bb5fd08 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 19:44:01 +0200 Subject: [PATCH 07/12] docs: fix example --- src/stdlib/parse_auditd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 27d314b41..93dffe3d8 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -178,7 +178,7 @@ impl Function for ParseAuditd { // TODO: add more examples based on tests &[Example { title: "parse auditd log", - source: "parse_auditd(\"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success\0x1dAUID=\\\"vrl\\\" UID=\\\"root\\\"\")", + source: "parse_auditd(\"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=success\x1dAUID=\\\"vrl\\\" UID=\\\"root\\\"\")", result: Ok(indoc! {r#" { "body": { From a64fe7ebebe5594e49b09e1d6df1e9cc776e0e2d Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 19:56:59 +0200 Subject: [PATCH 08/12] test(stdlib): improve test for parse_auditd --- src/stdlib/parse_auditd.rs | 74 ++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 43 deletions(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 93dffe3d8..e8fd00297 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -261,17 +261,6 @@ mod tests { use super::*; const ENRICHMENT_SEPARATOR: char = 0x1d as char; - // #[test] - // fn test_parse_auditd() { - // let line = r#"type=DAEMON_START msg=audit(1724423274.618:6439): op=start ver=4.0.2 format=enriched kernel=6.10.4-arch2-1 auid=1000 pid=1240242 uid=0 ses=2 res=successAUID="jorge" UID="root""#; - // let value = Value::from(line); - // println!("{}", parse_auditd(value).expect("expected ok ")); - - // let other_line = r#"type=SYSCALL msg=audit(1522927552.749:917): arch=c000003e syscall=2 success=yes exit=3 a0=7ffe2ce05793 a1=0 a2=1fffffffffff0000 a3=7ffe2ce043a0 items=1 ppid=2906 pid=4668 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts4 ses=1 comm="cat" exe="/bin/cat" key="passwd""#; - // let value = Value::from(other_line); - // println!("{}", parse_auditd(value).expect("expected ok ")); - // } - test_function![ parse_auditd => ParseAuditd; @@ -331,25 +320,24 @@ mod tests { // TODO: anonymize all those tests cases with random values while keeping actual syntax // TODO: include this in examples syscall { - args: func_args![ value: format!(r#"type=SYSCALL msg=audit(1615114232.375:15558): arch=c000003e syscall=59 success=yes exit=0 a0=63b29337fd18 a1=63b293387d58 a2=63b293375640 a3=fffffffffffff000 items=2 ppid=10883 pid=10884 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts1 ses=1 comm="whoami" exe="/usr/bin/whoami" key=(null){}ARCH=x86_64 SYSCALL=execve AUID="user" UID="root" GID="root" EUID="root" SUID="root" FSUID="root" EGID="root" SGID="root" FSGID="root""#, + args: func_args![ value: format!(r#"type=SYSCALL msg=audit(1724423274.618:6439): arch=c000003e syscall=59 success=yes exit=0 a0=123456789abcdef a1=123456789abcdef a2=123456789abcdef a3=123456789abcdef items=2 ppid=1240241 pid=1240242 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts1 ses=1 comm="whoami" exe="/usr/bin/whoami" key=(null){}ARCH=x86_64 SYSCALL=execve AUID="vrl" UID="root" GID="root" EUID="root" SUID="root" FSUID="root" EGID="root" SGID="root" FSGID="root""#, ENRICHMENT_SEPARATOR) ], want: Ok(btreemap! { "type" => "SYSCALL", - "timestamp" => DateTime::from_timestamp_millis(1_615_114_232_375), - "sequence" => 15558, - + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, "body" => btreemap! { "arch" => "0xc000003e", "syscall" => 59, "success" => "yes", "exit" => 0, - "a0" => "0x63b29337fd18", - "a1" => "0x63b293387d58", - "a2" => "0x63b293375640", - "a3" => "0xfffffffffffff000", + "a0" => "0x123456789abcdef", + "a1" => "0x123456789abcdef", + "a2" => "0x123456789abcdef", + "a3" => "0x123456789abcdef", "items" => 2, - "ppid" => 10_883, - "pid" => 10_884, + "ppid" => 1_240_241, + "pid" => 1_240_242, "auid" => 1000, "uid" => 0, "gid" => 0, @@ -363,12 +351,12 @@ mod tests { "ses" => 1, "comm" => "whoami", "exe" => "/usr/bin/whoami", - "key" => Value::Null, + "key" => Value::Null }, "enrichment" => btreemap! { "ARCH" => "x86_64", "SYSCALL" => "execve", - "AUID" => "user", + "AUID" => "vrl", "UID" => "root", "GID" => "root", "EUID" => "root", @@ -384,15 +372,15 @@ mod tests { // TODO: include this different types (denied Array) in examples avc_denied { - args: func_args![value: r#"type=AVC msg=audit(1631798689.083:65686): avc: denied { setuid } for pid=15381 comm="laurel" capability=7 scontext=system_u:system_r:auditd_t:s0 tcontext=system_u:system_r:auditd_t:s0 tclass=capability permissive=1"#], + args: func_args![value: r#"type=AVC msg=audit(1724423274.618:6439): avc: denied { setuid setuid2 setuid3 } for pid=1240242 comm="vrl" capability=7 scontext=system_u:system_r:auditd_t:s0 tcontext=system_u:system_r:auditd_t:s0 tclass=capability permissive=1"#], want: Ok(btreemap! { "type" => "AVC", - "timestamp" => DateTime::from_timestamp_millis(1_631_798_689_083), - "sequence" => 65686, + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, "body" => btreemap! { - "denied" => vec!["setuid"], - "pid" => 15_381, - "comm" => "laurel", + "denied" => vec!["setuid", "setuid2", "setuid3"], + "pid" => 1_240_242, + "comm" => "vrl", "capability" => 7, "scontext" => "system_u:system_r:auditd_t:s0", "tcontext" => "system_u:system_r:auditd_t:s0", @@ -404,16 +392,16 @@ mod tests { } avc_granted{ - args: func_args![value: r#"type=AVC msg=audit(1631870323.500:7098): avc: granted { setsecparam } for pid=11209 comm="tuned" scontext=system_u:system_r:tuned_t:s0 tcontext=system_u:object_r:security_t:s0 tclass=security"#], + args: func_args![value: r#"type=AVC msg=audit(1724423274.618:6439): avc: granted { setsecparam setsecparam2 setsecparam3} for pid=1240242 comm="vrl" scontext=system_u:system_r:vrl_t:s0 tcontext=system_u:object_r:security_t:s0 tclass=security"#], want: Ok(btreemap! { "type" => "AVC", - "timestamp" => DateTime::from_timestamp_millis(1_631_870_323_500), - "sequence" => 7098, + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, "body" => btreemap! { - "granted" => vec!["setsecparam"], - "pid" => 11_209, - "comm" => "tuned", - "scontext" => "system_u:system_r:tuned_t:s0", + "granted" => vec!["setsecparam","setsecparam2","setsecparam3"], + "pid" => 1_240_242, + "comm" => "vrl", + "scontext" => "system_u:system_r:vrl_t:s0", "tcontext" => "system_u:object_r:security_t:s0", "tclass" => "security" } @@ -422,23 +410,23 @@ mod tests { } user_acct { - args: func_args![value: format!(r#"type=USER_ACCT msg=audit(1724505830.648:19): pid=445523 uid=1000 auid=1000 ses=2 msg='op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="jorge" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success'{}UID="jorge" AUID="jorge""#, ENRICHMENT_SEPARATOR)], + args: func_args![value: format!(r#"type=USER_ACCT msg=audit(1724423274.618:6439): pid=1240242 uid=1000 auid=1000 ses=2 msg='op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="vrl" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success'{}UID="vrl" AUID="vrl""#, ENRICHMENT_SEPARATOR)], want: Ok(btreemap! { "type" => "USER_ACCT", - "timestamp" => DateTime::from_timestamp_millis(1_724_505_830_648), - "sequence" => 19, + "timestamp" => DateTime::from_timestamp_millis(1_724_423_274_618), + "sequence" => 6439, "body" => btreemap! { - "pid" => 445_523, + "pid" => 1_240_242, "uid" => 1_000, "auid" => 1_000, "ses" => 2, // TODO: this should be parsed into a nested object, but it is lack of implementation // from the linux-audit-parse crate - "msg" => r#"op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="jorge" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success"#, + "msg" => r#"op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="vrl" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success"#, }, "enrichment" => btreemap! { - "UID" => "jorge", - "AUID" => "jorge" + "UID" => "vrl", + "AUID" => "vrl" } }), tdef: type_def(), From f6904720013f35f33b93ab33a4aca78967358a3a Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 20:07:17 +0200 Subject: [PATCH 09/12] test(stdlib): remove comments from tests --- src/stdlib/parse_auditd.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index e8fd00297..13f13b6a6 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -317,8 +317,6 @@ mod tests { tdef: type_def(), } - // TODO: anonymize all those tests cases with random values while keeping actual syntax - // TODO: include this in examples syscall { args: func_args![ value: format!(r#"type=SYSCALL msg=audit(1724423274.618:6439): arch=c000003e syscall=59 success=yes exit=0 a0=123456789abcdef a1=123456789abcdef a2=123456789abcdef a3=123456789abcdef items=2 ppid=1240241 pid=1240242 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts1 ses=1 comm="whoami" exe="/usr/bin/whoami" key=(null){}ARCH=x86_64 SYSCALL=execve AUID="vrl" UID="root" GID="root" EUID="root" SUID="root" FSUID="root" EGID="root" SGID="root" FSGID="root""#, ENRICHMENT_SEPARATOR) ], @@ -370,7 +368,6 @@ mod tests { tdef: type_def(), } - // TODO: include this different types (denied Array) in examples avc_denied { args: func_args![value: r#"type=AVC msg=audit(1724423274.618:6439): avc: denied { setuid setuid2 setuid3 } for pid=1240242 comm="vrl" capability=7 scontext=system_u:system_r:auditd_t:s0 tcontext=system_u:system_r:auditd_t:s0 tclass=capability permissive=1"#], want: Ok(btreemap! { From 95d695110d6643e9bde1d52779174f311a8ffe54 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Fri, 30 Aug 2024 20:07:33 +0200 Subject: [PATCH 10/12] chore: update LICENSE-3rdparty.csv --- LICENSE-3rdparty.csv | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index b87964b88..787b2bca7 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -28,9 +28,10 @@ bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers borsh,https://github.com/near/borsh-rs,MIT OR Apache-2.0,Near Inc -borsh-derive,https://github.com/nearprotocol/borsh,Apache-2.0,Near Inc +borsh-derive,https://github.com/near/borsh-rs,Apache-2.0,Near Inc bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski +byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd @@ -47,7 +48,7 @@ cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Develop clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder Authors clap_derive,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_derive Authors -clap_lex,https://github.com/clap-rs/clap/tree/master/clap_lex,MIT OR Apache-2.0,The clap_lex Authors +clap_lex,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_lex Authors clipboard-win,https://github.com/DoumanAsh/clipboard-win,BSL-1.0,Douman codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zabarauskas colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors @@ -56,7 +57,7 @@ community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Ju concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum convert_case,https://github.com/rutrum/convert-case,MIT,Rutrum -core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors @@ -76,14 +77,13 @@ dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh D doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay -either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss +either,https://github.com/rayon-rs/either,MIT OR Apache-2.0,bluss encode_unicode,https://github.com/tormol/encode_unicode,Apache-2.0 OR MIT,Torbjørn Birch Moltu encoding_rs,https://github.com/hsivonen/encoding_rs,(Apache-2.0 OR MIT) AND BSD-3-Clause,Henri Sivonen equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors -errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,Chris Wong error-code,https://github.com/DoumanAsh/error-code,BSL-1.0,Douman -event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjepan Glavina -event-listener-strategy,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,John Nunley +event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " +event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " funty,https://github.com/myrrlyn/funty,MIT,myrrlyn @@ -98,8 +98,7 @@ grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats -hermit-abi,https://github.com/hermitcore/hermit-rs,MIT OR Apache-2.0,Stefan Lankes -hermit-abi,https://github.com/hermitcore/rusty-hermit,MIT OR Apache-2.0,Stefan Lankes +hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi hmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson @@ -110,8 +109,8 @@ indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap indoc,https://github.com/dtolnay/indoc,MIT OR Apache-2.0,David Tolnay influxdb-line-protocol,https://github.com/influxdata/influxdb_iox/tree/main/influxdb_line_protocol,MIT OR Apache-2.0,InfluxDB IOx Project Developers inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers -io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman is-terminal,https://github.com/sunfishcode/is-terminal,MIT,"softprops , Dan Gohman " +is_terminal_polyfill,https://github.com/polyfill-rs/is_terminal_polyfill,MIT OR Apache-2.0,The is_terminal_polyfill Authors itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase @@ -122,11 +121,11 @@ lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio -linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman +libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> +linux-audit-parser,https://github.com/hillu/linux-audit-parser-rs,LGPL-3.0-or-later,Hilko Bengen log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers md-5,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers memchr,https://github.com/BurntSushi/memchr,Unlicense OR MIT,"Andrew Gallant , bluss" -memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0,Alex Huszagh miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln " mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " @@ -138,7 +137,6 @@ num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Pro num-conv,https://github.com/jhpratt/num-conv,MIT OR Apache-2.0,Jacob Pratt num-integer,https://github.com/rust-num/num-integer,MIT OR Apache-2.0,The Rust Project Developers num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers -num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " objc-sys,https://github.com/madsmtm/objc2,MIT,Mads Marquart objc2,https://github.com/madsmtm/objc2,MIT,"Steven Sheldon, Mads Marquart " @@ -190,8 +188,7 @@ rend,https://github.com/djkoloski/rend,MIT,David Koloski rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski roxmltree,https://github.com/RazrFalcon/roxmltree,MIT OR Apache-2.0,Yevhenii Reizner rust_decimal,https://github.com/paupino/rust-decimal,MIT,Paul Mason -rustc-demangle,https://github.com/alexcrichton/rustc-demangle,MIT OR Apache-2.0,Alex Crichton -rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman , Jakub Konka " +rustc-demangle,https://github.com/rust-lang/rustc-demangle,MIT OR Apache-2.0,Alex Crichton rustversion,https://github.com/dtolnay/rustversion,MIT OR Apache-2.0,David Tolnay rustyline,https://github.com/kkawakam/rustyline,MIT,Katsu Kawakami ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay @@ -248,7 +245,7 @@ utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len , Dylan DPC, Hunar Roop Kahlon" valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " -vte_generate_state_changes,https://github.com/jwilm/vte,Apache-2.0 OR MIT,Christian Duerr +vte_generate_state_changes,https://github.com/alacritty/vte,Apache-2.0 OR MIT,Christian Duerr wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers wasm-bindgen,https://github.com/rustwasm/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-backend,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend,MIT OR Apache-2.0,The wasm-bindgen Developers @@ -263,6 +260,7 @@ windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto wyz,https://github.com/myrrlyn/wyz,MIT,myrrlyn +zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury From 0bd04e2e0571a3acd25164dbaea12ac32e5718c3 Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Mon, 21 Oct 2024 13:28:14 +0200 Subject: [PATCH 11/12] feat: update linux-audit-parser version --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/stdlib/parse_auditd.rs | 19 +++++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4e118986..5fd992f52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1515,9 +1515,9 @@ dependencies = [ [[package]] name = "linux-audit-parser" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a4056b37f64fbc83d1bc0ffdabd77a9238fadf46677cbc7f9e373e8389f9865" +checksum = "58f7d9e3689f07f34e7a1aeede0afbb749210c28ce4fcc362f8ff3bc5da63937" dependencies = [ "indexmap", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index bf1d1b559..34bee8201 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,7 +147,7 @@ influxdb-line-protocol = { version = "2.0.0", optional = true } indoc = {version = "2", optional = true } itertools = { version = "0.13", default-features = false, features=["use_std"], optional = true } lalrpop-util = { version = "0.21", optional = true } -linux-audit-parser = { version = "0.1.1", default-features = false, optional = true } +linux-audit-parser = { version = "0.2.0", default-features = false, optional = true } mlua = { version = "0.9", default-features = false, features = ["lua54", "send", "vendored"], optional = true} nom = { version = "7", default-features = false, features = ["std"], optional = true } once_cell = { version = "1", default-features = false, features = ["std"], optional = true } diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 13f13b6a6..21f944988 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -64,6 +64,7 @@ fn parse_auditd(bytes: Value) -> Resolved { log.insert("body".into(), Value::from(body)); + // TODO: should we downcase the keys of the enrichment so that they match its body counterparts? if !enrichment.is_empty() { log.insert("enrichment".into(), Value::from(enrichment)); } @@ -85,11 +86,6 @@ impl<'a> TryFrom> for Value { .collect::, _>>()?, ), AuditValue::Owned(string) => Value::from(Bytes::from(string)), - // Currently, AuditValue::Map is not generated from the parser (https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/value.rs#L70) - // but I think we should use that data type to represent the key-value pairs of the internal (nested) msg field - // as depicted here: https://github.com/vectordotdev/vrl/issues/66 - // For example, in the following linux-audit-parser test https://github.com/hillu/linux-audit-parser-rs/blob/d8c448c8d8227467b81cd5267790415b8b73f0cb/src/test.rs#L168 - // we see that the msg field is parsed as a string, although it could be key-value pairs AuditValue::Map(entries) => Value::from( entries .into_iter() @@ -417,9 +413,16 @@ mod tests { "uid" => 1_000, "auid" => 1_000, "ses" => 2, - // TODO: this should be parsed into a nested object, but it is lack of implementation - // from the linux-audit-parse crate - "msg" => r#"op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="vrl" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/1 res=success"#, + "msg"=> btreemap! { + "op" => "PAM:accounting", + "grantors" => "pam_unix,pam_permit,pam_time", + "acct" => "vrl", + "exe" => "/usr/bin/sudo", + "hostname" => Value::Null, + "addr" => Value::Null, + "terminal" => "/dev/pts/1", + "res" => "success" + } }, "enrichment" => btreemap! { "UID" => "vrl", From 7f52c6a03fff1fd413cac5e820a98a635c2aefae Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Mon, 21 Oct 2024 13:45:01 +0200 Subject: [PATCH 12/12] style: add whitespace to test name --- src/stdlib/parse_auditd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stdlib/parse_auditd.rs b/src/stdlib/parse_auditd.rs index 21f944988..36de3e39e 100644 --- a/src/stdlib/parse_auditd.rs +++ b/src/stdlib/parse_auditd.rs @@ -384,7 +384,7 @@ mod tests { tdef: type_def(), } - avc_granted{ + avc_granted { args: func_args![value: r#"type=AVC msg=audit(1724423274.618:6439): avc: granted { setsecparam setsecparam2 setsecparam3} for pid=1240242 comm="vrl" scontext=system_u:system_r:vrl_t:s0 tcontext=system_u:object_r:security_t:s0 tclass=security"#], want: Ok(btreemap! { "type" => "AVC",