implement adding older data, and filter data on the available data ra…

…nge when querying

src/binary/format.rs

@@ -109,7 +109,7 @@ impl<'data> TestAnalytics<'data> {
             let latest_test_timestamp = self.testdata[start_idx].last_timestamp;
 
             let today_offset = offset_from_today(latest_test_timestamp, self.timestamp);
-            let data_range = start_idx..start_idx + num_days;
+            let data_range = start_idx..start_idx + test.valid_data as usize;
             let adjusted_range =
                 adjust_selection_range(data_range, desired_range.clone(), today_offset);
 

src/binary/mod.rs

@@ -391,4 +391,52 @@ mod tests {
         let mut tests = parsed.tests(0..60, Some("non-existing")).unwrap();
         assert!(tests.next().is_none());
     }
+
+    #[test]
+    fn test_historic_data() {
+        let test = Testrun {
+            name: "abc".into(),
+            classname: "".into(),
+            duration: 1.0,
+            outcome: Outcome::Pass,
+            testsuite: "".into(),
+            failure_message: None,
+            filename: None,
+            build_url: None,
+            computed_name: None,
+        };
+
+        let mut writer = TestAnalyticsWriter::new(7);
+
+        let mut session = writer.start_session(3 * DAY, &[]);
+        session.insert(&test);
+        // insert data older than what is already in the file
+        let mut session = writer.start_session(1 * DAY, &[]);
+        session.insert(&test);
+
+        let mut buf = vec![];
+        writer.serialize(&mut buf).unwrap();
+
+        let parsed = TestAnalytics::parse(&buf, 4 * DAY).unwrap();
+
+        // we do not have any test data for "today"
+        let mut tests = parsed.tests(0..1, None).unwrap();
+        assert!(tests.next().is_none());
+
+        // when filtering for "yesterday", we get valid data
+        let mut tests = parsed.tests(1..2, None).unwrap();
+        let abc = tests.next().unwrap();
+        assert_eq!(abc.name().unwrap(), "abc");
+        assert!(tests.next().is_none());
+
+        // also when filtering for two days prior to that
+        let mut tests = parsed.tests(2..4, None).unwrap();
+        let abc = tests.next().unwrap();
+        assert_eq!(abc.name().unwrap(), "abc");
+        assert!(tests.next().is_none());
+
+        // but not when going further back in time
+        let mut tests = parsed.tests(5..7, None).unwrap();
+        assert!(tests.next().is_none());
+    }
 }

src/binary/raw.rs

@@ -23,15 +23,17 @@ pub struct Header {
 }
 unsafe impl Pod for Header {}
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy)]
 #[repr(C)]
 pub struct Test {
     /// Offset of the Testsuite name within the string table.
     pub testsuite_offset: u32,
     /// Offset of the Test name within the string table.
     pub name_offset: u32,
-
+    /// Offset of the Flag Set within the `FlagsSet` table.
     pub flag_set_offset: u32,
+    /// The number of valid data entries.
+    pub valid_data: u32,
 }
 unsafe impl Pod for Test {}
 
@@ -60,7 +62,7 @@ mod tests {
         assert_eq!(mem::size_of::<Header>(), 28);
         assert_eq!(mem::align_of::<Header>(), 4);
 
-        assert_eq!(mem::size_of::<Test>(), 8);
+        assert_eq!(mem::size_of::<Test>(), 16);
         assert_eq!(mem::align_of::<Test>(), 4);
 
         assert_eq!(mem::size_of::<TestData>(), 20);

src/binary/timestamps.rs

@@ -57,16 +57,10 @@ pub fn adjust_selection_range(
     desired_range: Range<usize>,
     today_offset: usize,
 ) -> Range<usize> {
-    let range_start = data_range
-        .start
-        .saturating_add(desired_range.start)
-        .saturating_sub(today_offset);
-    let range_end = data_range
-        .start
-        .saturating_add(desired_range.end)
-        .saturating_sub(today_offset);
-    let range_start = range_start.max(data_range.start);
-    let range_end = range_end.min(data_range.end);
+    let range_start = (data_range.start + desired_range.start).saturating_sub(today_offset);
+    let range_end = (data_range.start + desired_range.end).saturating_sub(today_offset);
+    let range_start = range_start.min(data_range.end).max(data_range.start);
+    let range_end = range_end.min(data_range.end).max(data_range.start);
     range_start..range_end
 }
 

src/binary/writer.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 use std::mem;
 
 use flagsset::FlagsSet;
-use indexmap::IndexSet;
+use indexmap::IndexMap;
 use raw::TestData;
 use timestamps::{adjust_selection_range, offset_from_today, shift_data};
 use watto::{Pod, StringTable};
@@ -15,6 +15,7 @@ use super::*;
 pub struct InsertSession<'writer> {
     writer: &'writer mut TestAnalyticsWriter,
 
+    timestamp: u32,
     flag_set_offset: u32,
 }
 
@@ -24,23 +25,50 @@ impl InsertSession<'_> {
     pub fn insert(&mut self, test: &testrun::Testrun) {
         let testsuite_offset = self.writer.string_table.insert(&test.testsuite) as u32;
         let name_offset = self.writer.string_table.insert(&test.name) as u32;
-        let (idx, inserted) = self.writer.tests.insert_full(raw::Test {
+        let key = TestKey {
+            testsuite_offset,
+            name_offset,
+            flag_set_offset: self.flag_set_offset,
+        };
+        let value = raw::Test {
             testsuite_offset,
             name_offset,
             flag_set_offset: self.flag_set_offset,
-        });
+            valid_data: 1,
+        };
+        let (idx, replaced) = self.writer.tests.insert_full(key, value);
 
-        let data_idx = idx * self.writer.num_days;
-        if inserted {
+        let mut data_idx = idx * self.writer.num_days;
+        if replaced.is_none() {
             let expected_size = self.writer.tests.len() * self.writer.num_days;
             self.writer
                 .testdata
                 .resize_with(expected_size, TestData::default);
         } else {
-            let range = data_idx..data_idx + self.writer.num_days;
-            let test_timestamp = self.writer.testdata[data_idx].last_timestamp;
-            let today_offset = offset_from_today(test_timestamp, self.writer.timestamp);
-            shift_data(&mut self.writer.testdata[range.clone()], today_offset);
+            let latest_timestamp = self.writer.testdata[data_idx].last_timestamp;
+
+            if latest_timestamp < self.timestamp {
+                // we are inserting newer data, so shift the existing data around
+                let today_offset = offset_from_today(latest_timestamp, self.timestamp);
+
+                let range = data_idx..data_idx + self.writer.num_days;
+                shift_data(&mut self.writer.testdata[range], today_offset);
+                extend_valid_data(
+                    &mut self.writer.tests[idx].valid_data,
+                    today_offset,
+                    self.writer.num_days,
+                );
+            } else {
+                // otherwise, we are inserting historic data, so adjust our `data_idx` accordingly
+                let today_offset = offset_from_today(self.timestamp, latest_timestamp);
+                if today_offset >= self.writer.num_days {
+                    return;
+                }
+                data_idx += today_offset;
+                self.writer.tests[idx].valid_data = self.writer.tests[idx]
+                    .valid_data
+                    .max(1 + today_offset as u32);
+            }
         }
 
         let testdata = &mut self.writer.testdata[data_idx];
@@ -59,6 +87,17 @@ impl InsertSession<'_> {
     }
 }
 
+fn extend_valid_data(valid_data: &mut u32, offset: usize, num_days: usize) {
+    *valid_data = (*valid_data as usize + offset).min(num_days) as u32;
+}
+
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+struct TestKey {
+    pub testsuite_offset: u32,
+    pub name_offset: u32,
+    pub flag_set_offset: u32,
+}
+
 /// The [`TestAnalytics`] File Writer.
 #[derive(Debug)]
 pub struct TestAnalyticsWriter {
@@ -69,7 +108,7 @@ pub struct TestAnalyticsWriter {
 
     timestamp: u32,
 
-    tests: IndexSet<raw::Test>,
+    tests: IndexMap<TestKey, raw::Test>,
     testdata: Vec<raw::TestData>,
 }
 
@@ -84,7 +123,7 @@ impl TestAnalyticsWriter {
 
             timestamp: 0,
 
-            tests: IndexSet::new(),
+            tests: IndexMap::new(),
             testdata: vec![],
         }
     }
@@ -96,13 +135,21 @@ impl TestAnalyticsWriter {
 
         InsertSession {
             writer: self,
+            timestamp: timestamp,
             flag_set_offset,
         }
     }
 
     /// Turns an existing parsed [`TestAnalytics`] file into a writer.
     pub fn from_existing_format(data: &TestAnalytics) -> Result<Self, TestAnalyticsError> {
-        let tests = IndexSet::from_iter(data.tests.iter().cloned());
+        let tests = IndexMap::from_iter(data.tests.iter().map(|test| {
+            let key = TestKey {
+                testsuite_offset: test.testsuite_offset,
+                name_offset: test.name_offset,
+                flag_set_offset: test.flag_set_offset,
+            };
+            (key, *test)
+        }));
 
         let string_table = StringTable::from_bytes(data.string_bytes)
             .map_err(|_| TestAnalyticsErrorKind::InvalidStringReference)?;
@@ -160,17 +207,24 @@ impl TestAnalyticsWriter {
                 .get(&test.flag_set_offset)
                 .ok_or(TestAnalyticsErrorKind::InvalidFlagSetReference)?;
 
-            let (idx, inserted) = writer.tests.insert_full(raw::Test {
+            let key = TestKey {
+                testsuite_offset,
+                name_offset,
+                flag_set_offset,
+            };
+            let value = raw::Test {
                 testsuite_offset,
                 name_offset,
                 flag_set_offset,
-            });
+                valid_data: 1,
+            };
+            let (idx, replaced) = writer.tests.insert_full(key, value);
 
             let data_idx = idx * writer.num_days;
             let smaller_idx = smaller_idx * smaller.header.num_days as usize;
             let smaller_timestamp = smaller.testdata[smaller_idx].last_timestamp;
 
-            let larger_timestamp = if inserted {
+            let larger_timestamp = if replaced.is_none() {
                 let expected_size = writer.tests.len() * writer.num_days;
                 writer
                     .testdata
@@ -187,6 +241,11 @@ impl TestAnalyticsWriter {
                 let range = data_idx..data_idx + writer.num_days;
 
                 shift_data(&mut writer.testdata[range], today_offset);
+                extend_valid_data(
+                    &mut writer.tests[idx].valid_data,
+                    today_offset,
+                    writer.num_days,
+                );
 
                 let smaller_range = adjust_selection_range(
                     smaller_idx..smaller_idx + smaller.header.num_days as usize,
@@ -209,6 +268,11 @@ impl TestAnalyticsWriter {
             let idx_start = data_idx + today_offset;
             let larger_range = idx_start..idx_start + overlap_len;
 
+            writer.tests[idx].valid_data = writer.tests[idx]
+                .valid_data
+                .max((larger_range.end - data_idx) as u32)
+                .min(writer.num_days as u32);
+
             let larger_data = &mut writer.testdata[larger_range];
             let smaller_data = &smaller.testdata[smaller_range];
 
@@ -272,7 +336,7 @@ impl TestAnalyticsWriter {
         self.tests.reserve(live_records);
         self.testdata.reserve(expected_size);
 
-        for ((old_idx, test), record_live) in tests.iter().enumerate().zip(record_liveness) {
+        for ((old_idx, test), record_live) in tests.values().enumerate().zip(record_liveness) {
             if !record_live {
                 continue;
             }
@@ -295,12 +359,19 @@ impl TestAnalyticsWriter {
 
             let testsuite_offset = self.string_table.insert(testsuite) as u32;
             let name_offset = self.string_table.insert(name) as u32;
-            let (_new_idx, inserted) = self.tests.insert_full(raw::Test {
+            let key = TestKey {
                 testsuite_offset,
                 name_offset,
                 flag_set_offset,
-            });
-            assert!(inserted); // the records are already unique, and we re-insert those
+            };
+            let value = raw::Test {
+                testsuite_offset,
+                name_offset,
+                flag_set_offset,
+                valid_data: test.valid_data.max(num_days as u32),
+            };
+            let (_new_idx, replaced) = self.tests.insert_full(key, value);
+            assert!(replaced.is_none()); // the records are already unique, and we re-insert those
 
             let overlap_days = num_days.min(self.num_days);
             let old_idx = old_idx * num_days;
@@ -339,7 +410,7 @@ impl TestAnalyticsWriter {
 
         writer.write_all(header.as_bytes())?;
 
-        for test in self.tests.into_iter() {
+        for test in self.tests.into_values() {
             writer.write_all(test.as_bytes())?;
         }