Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Time releated fixes #466

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Time releated fixes #466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
03055ef
filterx: fix datetime conversion to string
bshifter Jan 24, 2025
3dcb830
timeutils: prepare nano sec precision handling
bshifter Jan 24, 2025
62cfb13
time: fix unix_time_to_unix_epoch calculation
bshifter Jan 24, 2025
ef83f0b
otel: fix timestamp calculation for nanosec precision otel fields
bshifter Jan 24, 2025
3a63778
time: add UnixTime dumper fn for debugging purposes
bshifter Jan 24, 2025
357652f
time: Add WallClockTime dumper function for debugging purposes
bshifter Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/filterx/expr-comparison.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ _convert_filterx_object_to_generic_number(FilterXObject *obj, GenericNumber *gn)
UnixTime utime;
if (filterx_object_extract_datetime(obj, &utime))
{
guint64 unix_epoch = unix_time_to_unix_epoch(utime);
guint64 unix_epoch = unix_time_to_unix_epoch_usec(utime);
gn_set_int64(gn, (gint64) MIN(unix_epoch, G_MAXINT64));
return;
}
Expand Down
6 changes: 2 additions & 4 deletions lib/filterx/object-datetime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ _convert_unix_time_to_string(const UnixTime *ut, GString *result, gboolean inclu
{
if (ut->ut_gmtoff >= 0)
g_string_append_c(result, '+');
else
g_string_append_c(result, '-');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good catch. a unit test for this would be great somewhere.


format_int64_padded(result, 2, '0', 10, ut->ut_gmtoff / 3600);
g_string_append_c(result, ':');
Expand Down Expand Up @@ -136,14 +134,14 @@ filterx_typecast_datetime(FilterXExpr *s, FilterXObject *args[], gsize args_len)
gint64 i;
if (filterx_object_extract_integer(object, &i))
{
UnixTime ut = unix_time_from_unix_epoch((guint64) MAX(i, 0));
UnixTime ut = unix_time_from_unix_epoch_usec((guint64) MAX(i, 0));
return filterx_datetime_new(&ut);
}

gdouble d;
if (filterx_object_extract_double(object, &d))
{
UnixTime ut = unix_time_from_unix_epoch((gint64)(d * USEC_PER_SEC));
UnixTime ut = unix_time_from_unix_epoch_usec((gint64)(d * USEC_PER_SEC));
return filterx_datetime_new(&ut);
}

Expand Down
6 changes: 3 additions & 3 deletions lib/filterx/tests/test_expr_comparison.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ Test(expr_comparison, test_integer_to_bool_num_based_comparison)
Test(expr_comparison, test_integer_to_datetime_num_based_comparison)
{
UnixTime ut = { .ut_sec = 1701350398, .ut_usec = 123000, .ut_gmtoff = 3600 };
guint64 epoch = unix_time_to_unix_epoch(ut);
guint64 epoch = unix_time_to_unix_epoch_usec(ut);
_assert_comparison(filterx_integer_new((uint64_t)(epoch)), filterx_datetime_new(&ut), FCMPX_EQ | FCMPX_NUM_BASED, TRUE);
_assert_comparison(filterx_integer_new((uint64_t)(epoch)), filterx_datetime_new(&ut), FCMPX_LT | FCMPX_NUM_BASED,
FALSE);
Expand Down Expand Up @@ -277,7 +277,7 @@ Test(expr_comparison, test_double_to_bool_num_based_comparison)
Test(expr_comparison, test_double_to_datetime_num_based_comparison)
{
UnixTime ut = { .ut_sec = 1701350398, .ut_usec = 123000, .ut_gmtoff = 3600 };
guint64 epoch = unix_time_to_unix_epoch(ut);
guint64 epoch = unix_time_to_unix_epoch_usec(ut);
_assert_comparison(filterx_double_new((double)(epoch)), filterx_datetime_new(&ut), FCMPX_EQ | FCMPX_NUM_BASED, TRUE);
_assert_comparison(filterx_double_new((double)(epoch)), filterx_datetime_new(&ut), FCMPX_LT | FCMPX_NUM_BASED, FALSE);
_assert_comparison(filterx_double_new((double)(epoch)), filterx_datetime_new(&ut), FCMPX_GT | FCMPX_NUM_BASED, FALSE);
Expand Down Expand Up @@ -605,7 +605,7 @@ Test(expr_comparison, test_string_cases_type_aware_comparison)
Test(expr_comparison, test_numerical_fallback_type_aware_comparison)
{
UnixTime ut = { .ut_sec = 1701350398, .ut_usec = 123000, .ut_gmtoff = 3600 };
guint64 epoch = unix_time_to_unix_epoch(ut);
guint64 epoch = unix_time_to_unix_epoch_usec(ut);
_assert_comparison(filterx_integer_new(6), filterx_integer_new(6), FCMPX_EQ | FCMPX_TYPE_AWARE, TRUE);
_assert_comparison(filterx_integer_new(6), filterx_double_new(6.0), FCMPX_EQ | FCMPX_TYPE_AWARE, TRUE);
_assert_comparison(filterx_integer_new(3), filterx_string_new("3", 1), FCMPX_EQ | FCMPX_TYPE_AWARE, TRUE);
Expand Down
10 changes: 5 additions & 5 deletions lib/filterx/tests/test_expr_plus.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ Test(expr_plus, test_string_add_wrong_type)

Test(expr_plus, test_datetime_add_datetime)
{
UnixTime lhs_time = unix_time_from_unix_epoch(TEST_EPOCH);
UnixTime rhs_time = unix_time_from_unix_epoch(3600000000); // 1 h
UnixTime lhs_time = unix_time_from_unix_epoch_usec(TEST_EPOCH);
UnixTime rhs_time = unix_time_from_unix_epoch_usec(3600000000); // 1 h

FilterXExpr *lhs = filterx_literal_new(filterx_datetime_new(&lhs_time));
FilterXExpr *rhs = filterx_literal_new(filterx_datetime_new(&rhs_time));
Expand All @@ -107,7 +107,7 @@ Test(expr_plus, test_datetime_add_datetime)

Test(expr_plus, test_datetime_add_integer)
{
UnixTime lhs_time = unix_time_from_unix_epoch(TEST_EPOCH);
UnixTime lhs_time = unix_time_from_unix_epoch_usec(TEST_EPOCH);

FilterXExpr *lhs = filterx_literal_new(filterx_datetime_new(&lhs_time));
FilterXExpr *rhs = filterx_literal_new(filterx_integer_new(3600000000)); // 1h in usec
Expand All @@ -130,7 +130,7 @@ Test(expr_plus, test_datetime_add_integer)

Test(expr_plus, test_datetime_add_double)
{
UnixTime lhs_time = unix_time_from_unix_epoch(TEST_EPOCH);
UnixTime lhs_time = unix_time_from_unix_epoch_usec(TEST_EPOCH);

FilterXExpr *lhs = filterx_literal_new(filterx_datetime_new(&lhs_time));
FilterXExpr *rhs = filterx_literal_new(filterx_double_new(3600.0)); // 1h in sec
Expand All @@ -153,7 +153,7 @@ Test(expr_plus, test_datetime_add_double)

Test(expr_plus, test_datetime_add_wrong_type)
{
UnixTime lhs_time = unix_time_from_unix_epoch(TEST_EPOCH);
UnixTime lhs_time = unix_time_from_unix_epoch_usec(TEST_EPOCH);
FilterXExpr *lhs = filterx_literal_new(filterx_datetime_new(&lhs_time));
FilterXExpr *rhs = filterx_literal_new(filterx_null_new());

Expand Down
25 changes: 24 additions & 1 deletion lib/filterx/tests/test_object_datetime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ Test(filterx_datetime, test_filterx_datetime_typecast_from_datetime)

Test(filterx_datetime, test_filterx_datetime_repr_method)
{
UnixTime ut = unix_time_from_unix_epoch(3600000000);
UnixTime ut = unix_time_from_unix_epoch_usec(3600000000);
cr_assert(ut.ut_gmtoff == 0);
cr_assert(ut.ut_usec == 0);
cr_assert(ut.ut_sec == 3600);
Expand All @@ -161,6 +161,29 @@ Test(filterx_datetime, test_filterx_datetime_repr_method)
cr_assert_str_eq(repr->str, "1970-01-01T01:00:00.000+00:00");
}

Test(filterx_datetime, test_filterx_datetime_marshal)
{
UnixTime ut = { .ut_sec = 1701350398, .ut_usec = 123000, .ut_gmtoff = 3600 };
FilterXObject *obj = filterx_datetime_new(&ut);
cr_assert_not_null(obj);

LogMessageValueType lmvt;
GString *repr = scratch_buffers_alloc();
cr_assert(filterx_object_marshal(obj, repr, &lmvt));
cr_assert_str_eq(repr->str, "1701350398.123000+01:00");
}

Test(filterx_datetime, test_filterx_datetime_marshal_negative_offset)
{
UnixTime ut = { .ut_sec = 1701350398, .ut_usec = 123000, .ut_gmtoff = -10800 };
FilterXObject *obj = filterx_datetime_new(&ut);
cr_assert_not_null(obj);

LogMessageValueType lmvt;
GString *repr = scratch_buffers_alloc();
cr_assert(filterx_object_marshal(obj, repr, &lmvt));
cr_assert_str_eq(repr->str, "1701350398.123000-3:00");
}

Test(filterx_datetime, test_filterx_datetime_repr)
{
Expand Down
3 changes: 2 additions & 1 deletion lib/timeutils/misc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,14 @@
# define NSEC_PER_SEC 1000000000L
# define USEC_PER_SEC 1000000L
# define MSEC_PER_SEC 1000L
# define NSEC_PER_USEC 1000L
#endif

#define MSEC_TO_NSEC(msec) ((msec) * NSEC_PER_SEC / MSEC_PER_SEC)
#define MSEC_TO_USEC(msec) ((msec) * USEC_PER_SEC / MSEC_PER_SEC)

#define USEC_TO_NSEC(usec) ((usec) * NSEC_PER_SEC / USEC_PER_SEC)
#define USEC_TO_USEC(usec) ((usec) * USEC_PER_SEC / USEC_PER_SEC)
#define NSEC_TO_USEC(nsec) ((nsec) / NSEC_PER_USEC)

gboolean check_nanosleep(void);

Expand Down
52 changes: 47 additions & 5 deletions lib/timeutils/unixtime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,19 +359,30 @@ _unix_time_parts_from_unix_epoch(guint64 unix_epoch, gint64 *secs, guint32 *usec
}

UnixTime
unix_time_from_unix_epoch(guint64 unix_epoch)
unix_time_from_unix_epoch_usec(guint64 unix_epoch_usec)
{
UnixTime ut = UNIX_TIME_INIT;
ut.ut_gmtoff = 0;
_unix_time_parts_from_unix_epoch(unix_epoch, &ut.ut_sec, &ut.ut_usec);
_unix_time_parts_from_unix_epoch(unix_epoch_usec, &ut.ut_sec, &ut.ut_usec);
return ut;
}

guint64
unix_time_to_unix_epoch(const UnixTime ut)
unix_time_to_unix_epoch_usec(const UnixTime ut)
{
gint32 gmtoff = (ut.ut_gmtoff != - 1) ? ut.ut_gmtoff : 0;
return (guint64)((ut.ut_sec + gmtoff) * USEC_PER_SEC + ut.ut_usec);
return (guint64)(ut.ut_sec * USEC_PER_SEC + ut.ut_usec);
}

UnixTime
unix_time_from_unix_epoch_nsec(guint64 unix_epoch_nsec)
{
return unix_time_from_unix_epoch_usec(NSEC_TO_USEC(unix_epoch_nsec));
}

guint64
unix_time_to_unix_epoch_nsec(const UnixTime ut)
{
return unix_time_to_unix_epoch_usec(ut) * NSEC_PER_USEC;
}

UnixTime
Expand All @@ -389,3 +400,34 @@ unix_time_add_duration(UnixTime time, guint64 duration)
return ut;

}

void
dump_unix_time(const UnixTime *ut, GString *output)
{
if (!ut || !output)
return;

g_string_append_printf(output, "UnixTime:\n");
g_string_append_printf(output, " Seconds since epoch: %" G_GINT64_FORMAT "\n", ut->ut_sec);
g_string_append_printf(output, " Microseconds: %u\n", ut->ut_usec);
g_string_append_printf(output, " GMT Offset (seconds): %d\n", ut->ut_gmtoff);

time_t utc_time = (time_t)ut->ut_sec;
time_t local_time = utc_time + ut->ut_gmtoff;

struct tm utc_tm, local_tm;

gmtime_r(&utc_time, &utc_tm);
localtime_r(&local_time, &local_tm);

char utc_time_str[64];
char local_time_str[64];

strftime(utc_time_str, sizeof(utc_time_str), "%Y-%m-%d %H:%M:%S", &utc_tm);
strftime(local_time_str, sizeof(local_time_str), "%Y-%m-%d %H:%M:%S", &local_tm);

g_string_append_printf(output, " Human-Readable Time:\n");
g_string_append_printf(output, " UTC Time: %s.%06u\n", utc_time_str, ut->ut_usec);
g_string_append_printf(output, " Local Time: %s.%06u (GMT offset: %d seconds)\n",
local_time_str, ut->ut_usec, ut->ut_gmtoff);
}
7 changes: 5 additions & 2 deletions lib/timeutils/unixtime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,11 @@ gint64 unix_time_diff_in_seconds(const UnixTime *a, const UnixTime *b);
gint64 unix_time_diff_in_msec(const UnixTime *a, const UnixTime *b);

struct timeval timeval_from_unix_time(UnixTime *ut);
UnixTime unix_time_from_unix_epoch(guint64 unix_epoch);
guint64 unix_time_to_unix_epoch(const UnixTime ut);
UnixTime unix_time_from_unix_epoch_usec(guint64 unix_epoch);
guint64 unix_time_to_unix_epoch_usec(const UnixTime ut);
UnixTime unix_time_from_unix_epoch_nsec(guint64 unix_epoch_nsec);
guint64 unix_time_to_unix_epoch_nsec(const UnixTime ut);
UnixTime unix_time_add_duration(UnixTime time, guint64 duration);
void dump_unix_time(const UnixTime *ut, GString *output);

#endif
22 changes: 22 additions & 0 deletions lib/timeutils/wallclocktime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1005,3 +1005,25 @@ find_string(const unsigned char *bp, int *tgt, const char *const *n1,
/* Nothing matched */
return NULL;
}

void
dump_wall_clock_time(const WallClockTime *wct, GString *output)
{
if (!wct || !output)
return;

g_string_append_printf(output, "WallClockTime:\n");
g_string_append_printf(output, " Year: %d\n", wct->wct_year + 1900); // tm_year is years since 1900
g_string_append_printf(output, " Month: %d\n", wct->wct_mon + 1); // tm_mon is zero-based
g_string_append_printf(output, " Day of Month: %d\n", wct->wct_mday);
g_string_append_printf(output, " Weekday: %d\n", wct->wct_wday);
g_string_append_printf(output, " Day of Year: %d\n", wct->wct_yday);
g_string_append_printf(output, " Hour: %d\n", wct->wct_hour);
g_string_append_printf(output, " Minute: %d\n", wct->wct_min);
g_string_append_printf(output, " Second: %d\n", wct->wct_sec);
g_string_append_printf(output, " Microsecond: %d\n", wct->wct_usec);
g_string_append_printf(output, " DST: %d\n", wct->wct_isdst);

g_string_append_printf(output, " GMT Offset: %ld\n", wct->wct_gmtoff);
g_string_append_printf(output, " Timezone: %s\n", wct->wct_zone ? wct->wct_zone : "(null)");
}
1 change: 1 addition & 0 deletions lib/timeutils/wallclocktime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,5 +136,6 @@ void wall_clock_time_unset(WallClockTime *wct);
gchar *wall_clock_time_strptime(WallClockTime *wct, const gchar *format, const gchar *input);
void wall_clock_time_guess_missing_year(WallClockTime *self);
void wall_clock_time_guess_missing_fields(WallClockTime *self);
void dump_wall_clock_time(const WallClockTime *wct, GString *output);

#endif
6 changes: 4 additions & 2 deletions modules/grpc/otel/filterx/otel-field.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "object-otel-array.hpp"

#include "compat/cpp-start.h"

#include "filterx/filterx-object.h"
#include "filterx/object-extractor.h"
#include "filterx/object-string.h"
Expand All @@ -39,6 +40,7 @@
#include "filterx/object-null.h"
#include "filterx/filterx-object-istype.h"
#include "filterx/filterx-ref.h"

#include "compat/cpp-end.h"

#include "opentelemetry/proto/logs/v1/logs.pb.h"
Expand Down Expand Up @@ -269,7 +271,7 @@ class OtelDatetimeConverter : public ProtobufField
FilterXObject *FilterXObjectGetter(Message *message, ProtoReflectors reflectors)
{
uint64_t val = reflectors.reflection->GetUInt64(*message, reflectors.fieldDescriptor);
UnixTime utime = unix_time_from_unix_epoch(val);
UnixTime utime = unix_time_from_unix_epoch_nsec(val);
return filterx_datetime_new(&utime);
}
bool FilterXObjectSetter(Message *message, ProtoReflectors reflectors, FilterXObject *object,
Expand All @@ -278,7 +280,7 @@ class OtelDatetimeConverter : public ProtobufField
UnixTime utime;
if (filterx_object_extract_datetime(object, &utime))
{
uint64_t unix_epoch = unix_time_to_unix_epoch(utime);
uint64_t unix_epoch = unix_time_to_unix_epoch_nsec(utime);
reflectors.reflection->SetUInt64(message, reflectors.fieldDescriptor, unix_epoch);
return true;
}
Expand Down
4 changes: 2 additions & 2 deletions modules/grpc/otel/filterx/protobuf-field.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ class i64Field : public ProtobufField
UnixTime utime;
if (filterx_object_extract_datetime(object, &utime))
{
uint64_t unix_epoch = unix_time_to_unix_epoch(utime);
uint64_t unix_epoch = unix_time_to_unix_epoch_usec(utime);
reflectors.reflection->SetInt64(message, reflectors.fieldDescriptor, (int64_t)(unix_epoch));
return true;
}
Expand Down Expand Up @@ -189,7 +189,7 @@ class u64Field : public ProtobufField
UnixTime utime;
if (filterx_object_extract_datetime(object, &utime))
{
uint64_t unix_epoch = unix_time_to_unix_epoch(utime);
uint64_t unix_epoch = unix_time_to_unix_epoch_usec(utime);
reflectors.reflection->SetUInt64(message, reflectors.fieldDescriptor, unix_epoch);
return true;
}
Expand Down
12 changes: 10 additions & 2 deletions modules/grpc/otel/tests/test-otel-filterx.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "filterx/object-string.h"
#include "filterx/object-primitive.h"
#include "filterx/filterx-object-istype.h"
#include "filterx/object-datetime.h"
#include "apphook.h"
#include "cfg.h"
#include "compat/cpp-end.h"
Expand Down Expand Up @@ -279,19 +280,26 @@ Test(otel_filterx, logrecord_iter)
FilterXObject *body = filterx_string_new("body", -1);
FilterXObject *body_val = filterx_string_new("body_val", -1);
FilterXObject *time_unix_nano = filterx_string_new("time_unix_nano", -1);
FilterXObject *time_unix_nano_val = filterx_integer_new(123);
FilterXObject *time_unix_nano_val = filterx_integer_new(123000);
FilterXObject *observed_time_unix_nano = filterx_string_new("observed_time_unix_nano", -1);
UnixTime ut = {.ut_sec = 1712736550, .ut_usec = 1000, .ut_gmtoff = -10800};
FilterXObject *observed_time_unix_nano_val = filterx_datetime_new(&ut);

cr_assert(filterx_object_set_subscript(logrecord, body, &body_val));
cr_assert(filterx_object_set_subscript(logrecord, time_unix_nano, &time_unix_nano_val));
cr_assert(filterx_object_set_subscript(logrecord, observed_time_unix_nano, &observed_time_unix_nano_val));

GString *output = g_string_new(NULL);
cr_assert(filterx_dict_iter(logrecord, _append_to_str, output));

cr_assert_str_eq(output->str, "time_unix_nano\n0.000123+00:00\nbody\nbody_val\n");
cr_assert_str_eq(output->str,
"time_unix_nano\n0.000123+00:00\nbody\nbody_val\nobserved_time_unix_nano\n1712736550.001000+00:00\n");

g_string_free(output, TRUE);
filterx_object_unref(time_unix_nano);
filterx_object_unref(time_unix_nano_val);
filterx_object_unref(observed_time_unix_nano);
filterx_object_unref(observed_time_unix_nano_val);
filterx_object_unref(body);
filterx_object_unref(body_val);
filterx_object_unref(logrecord);
Expand Down
8 changes: 4 additions & 4 deletions tests/light/functional_tests/filterx/test_filterx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def test_otel_logrecord_datetime_setter_getter(config, syslog_ng):

assert file_true.get_stats()["processed"] == 1
assert "processed" not in file_false.get_stats()
assert file_true.read_log() == "1701353998.123000+00:00\n"
assert file_true.read_log() == "1701350398.123000+00:00\n"


def test_otel_logrecord_body_string_setter_getter(config, syslog_ng):
Expand Down Expand Up @@ -210,7 +210,7 @@ def test_otel_logrecord_body_datetime_setter_getter(config, syslog_ng):
# does not distinguish int_value and datetime.
assert file_true.get_stats()["processed"] == 1
assert "processed" not in file_false.get_stats()
assert file_true.read_log() == "1701353998123000\n"
assert file_true.read_log() == "1701350398123000\n"


def test_otel_logrecord_body_bytes_setter_getter(config, syslog_ng):
Expand Down Expand Up @@ -317,7 +317,7 @@ def test_otel_resource_scope_log_to_json(config, syslog_ng):
log = otel_logrecord();
log += {
"body": "fit",
"time_unix_nano": 123456789,
"time_unix_nano": 123456789000,
"attributes": {
"answer": 42,
"cool": true
Expand Down Expand Up @@ -404,7 +404,7 @@ def test_json_to_otel_resource_scope_log(config, syslog_ng):
log_js = json();
log_js += {
"body": "fit",
"time_unix_nano": 123456789,
"time_unix_nano": 123456789000,
"attributes": {
"answer": 42,
"cool": true
Expand Down
Loading