Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(parse_groks): expose internal errors #1113

Merged
merged 4 commits into from
Nov 12, 2024
Merged

chore(parse_groks): expose internal errors #1113

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3c84964
fix(parse_groks): surface grok filter errors
neuronull Nov 7, 2024
2848428
differentiate fatal and recoverable errors
neuronull Nov 8, 2024
936f032
rename to InternalErrors
neuronull Nov 12, 2024
77aa8fd
Merge branch 'main' into neuronull/OPA-2627_parse_grok_surface_filter…
neuronull Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/datadog/grok/filters/keyvalue.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use ordered_float::NotNan;
use super::super::{
ast::{Function, FunctionArgument},
grok_filter::GrokFilter,
parse_grok::Error as GrokRuntimeError,
parse_grok::RecoverableGrokError,
parse_grok_rules::Error as GrokStaticError,
};

Expand Down Expand Up @@ -166,7 +166,7 @@ pub fn regex_from_config(
}

impl KeyValueFilter {
pub fn apply_filter(&self, value: &Value) -> Result<Value, GrokRuntimeError> {
pub fn apply_filter(&self, value: &Value) -> Result<Value, RecoverableGrokError> {
match value {
Value::Bytes(bytes) => {
let mut result = Value::Object(BTreeMap::default());
Expand All @@ -176,7 +176,7 @@ impl KeyValueFilter {
});
Ok(result)
}
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
self.to_string(),
value.to_string(),
)),
Expand Down
8 changes: 3 additions & 5 deletions src/datadog/grok/grok.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use std::collections::{btree_map, BTreeMap};
use std::panic;
use std::sync::Arc;

use super::parse_grok::Error as GrokRuntimeError;
use super::parse_grok::FatalGrokError;

use onig::{Captures, Regex};
use thiserror::Error;
Expand Down Expand Up @@ -113,16 +113,14 @@ impl Pattern {
pub fn match_against<'a>(
&'a self,
text: &'a str,
) -> Result<Option<Matches<'a>>, GrokRuntimeError> {
) -> Result<Option<Matches<'a>>, FatalGrokError> {
let result = panic::catch_unwind(|| self.regex.captures(text));

match result {
Ok(Some(cap)) => Ok(Some(Matches::new(cap, &self.names))),
Ok(None) => Ok(None),
// https://github.com/rust-onig/rust-onig/issues/178
Err(_) => Err(GrokRuntimeError::FailedToMatch(
"Regex search error in the underlying engine".into(),
)),
Err(_) => Err(FatalGrokError::RegexEngineError),
}
}
}
Expand Down
42 changes: 24 additions & 18 deletions src/datadog/grok/grok_filter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use super::{
ast::{Function, FunctionArgument},
filters::{array, keyvalue, keyvalue::KeyValueFilter},
matchers::date::{apply_date_filter, DateFilter},
parse_grok::Error as GrokRuntimeError,
parse_grok::RecoverableGrokError,
parse_grok_rules::Error as GrokStaticError,
};

Expand Down Expand Up @@ -119,16 +119,16 @@ impl TryFrom<&Function> for GrokFilter {

/// Applies a given Grok filter to the value and returns the result or error.
/// For detailed description and examples of specific filters check out https://docs.datadoghq.com/logs/log_configuration/parsing/?tab=filters
pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRuntimeError> {
pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, RecoverableGrokError> {
match filter {
GrokFilter::Integer => match value {
Value::Bytes(v) => Ok(String::from_utf8_lossy(v)
.parse::<i64>()
.map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(filter.to_string(), value.to_string())
})?
.into()),
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -137,10 +137,10 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
Value::Bytes(v) => Ok(String::from_utf8_lossy(v)
.parse::<f64>()
.map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(filter.to_string(), value.to_string())
})
.map(|f| (f as i64).into())?),
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -149,7 +149,10 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
Value::Bytes(v) => {
let v = Ok(Value::from_f64_or_zero(
String::from_utf8_lossy(v).parse::<f64>().map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)
})?,
));
match v {
Expand All @@ -159,7 +162,7 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
_ => v,
}
}
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -175,11 +178,14 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
)),
Value::Bytes(v) => {
let v = String::from_utf8_lossy(v).parse::<f64>().map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)
})?;
Ok(Value::Float(NotNan::new(v * scale_factor).expect("NaN")))
}
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand Down Expand Up @@ -233,7 +239,7 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
Ok(value.to_owned())
}
}
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -249,7 +255,7 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
delimiter.as_ref().map(|s| s.as_str()),
)
.map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(filter.to_string(), value.to_string())
})
.and_then(|values| {
if let Some(value_filter) = value_filter.as_ref() {
Expand All @@ -262,7 +268,7 @@ pub fn apply_filter(value: &Value, filter: &GrokFilter) -> Result<Value, GrokRun
}
Ok(values.into())
}),
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -274,10 +280,10 @@ fn parse_value<V: Into<Value>>(
value: &Value,
filter: &GrokFilter,
parse: impl Fn(&Bytes) -> V,
) -> Result<Value, GrokRuntimeError> {
) -> Result<Value, RecoverableGrokError> {
match value {
Value::Bytes(bytes) => Ok(parse(bytes).into()),
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand All @@ -288,14 +294,14 @@ fn parse_value_error_prone<V: Into<Value>, E: std::error::Error>(
value: &Value,
filter: &GrokFilter,
parse: impl Fn(&Bytes) -> Result<V, E>,
) -> Result<Value, GrokRuntimeError> {
) -> Result<Value, RecoverableGrokError> {
match value {
Value::Bytes(bytes) => parse(bytes)
.map_err(|_e| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(filter.to_string(), value.to_string())
})
.map(Into::into),
_ => Err(GrokRuntimeError::FailedToApplyFilter(
_ => Err(RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
value.to_string(),
)),
Expand Down
28 changes: 17 additions & 11 deletions src/datadog/grok/matchers/date.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use peeking_take_while::PeekableExt;
use regex::Regex;
use tracing::warn;

use super::super::parse_grok::Error as GrokRuntimeError;
use super::super::parse_grok::RecoverableGrokError;

/// converts Joda time format to strptime format
pub fn convert_time_format(format: &str) -> Result<String, String> {
Expand Down Expand Up @@ -242,9 +242,12 @@ pub fn time_format_to_regex(format: &str, with_captures: bool) -> Result<RegexRe
Ok(RegexResult { regex, with_tz })
}

pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, GrokRuntimeError> {
pub fn apply_date_filter(
value: &Value,
filter: &DateFilter,
) -> Result<Value, RecoverableGrokError> {
let original_value = String::from_utf8_lossy(value.as_bytes().ok_or_else(|| {
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), value.to_string())
RecoverableGrokError::FailedToApplyFilter(filter.to_string(), value.to_string())
})?);
let (strp_format, mut datetime) =
adjust_strp_format_and_value(&filter.strp_format, &original_value);
Expand All @@ -264,13 +267,16 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
let tz = tz.as_str();
let tz: Tz = tz.parse().map_err(|error| {
warn!(message = "Error parsing tz", tz = %tz, % error);
GrokRuntimeError::FailedToApplyFilter(filter.to_string(), original_value.to_string())
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
})?;
replace_sec_fraction_with_dot(filter, &mut datetime);
let naive_date = NaiveDateTime::parse_from_str(&datetime, &strp_format).map_err(|error|
{
warn!(message = "Error parsing date", value = %original_value, format = %strp_format, % error);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand All @@ -279,7 +285,7 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
.from_local_datetime(&naive_date)
.single()
.ok_or_else(|| {
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand All @@ -293,7 +299,7 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
// parse as a tz-aware complete date/time
let timestamp = DateTime::parse_from_str(&datetime, &strp_format).map_err(|error| {
warn!(message = "Error parsing date", date = %original_value, % error);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand All @@ -304,14 +310,14 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
if let Some(tz) = &filter.target_tz {
let tzs = parse_timezone(tz).map_err(|error| {
warn!(message = "Error parsing tz", tz = %tz, % error);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
})?;
let dt = tzs.from_local_datetime(&dt).single().ok_or_else(|| {
warn!(message = "Error parsing date", date = %original_value);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand All @@ -334,7 +340,7 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
// try parsing as a naive date
let nd = NaiveDate::parse_from_str(&datetime, &strp_format).map_err(|error| {
warn!(message = "Error parsing date", date = %original_value, % error);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand All @@ -347,7 +353,7 @@ pub fn apply_date_filter(value: &Value, filter: &DateFilter) -> Result<Value, Gr
.single()
.ok_or_else(|| {
warn!(message = "Error parsing date", date = %original_value);
GrokRuntimeError::FailedToApplyFilter(
RecoverableGrokError::FailedToApplyFilter(
filter.to_string(),
original_value.to_string(),
)
Expand Down
Loading
Loading