Add ECH and OONI Run related columns to obs_web, obs_web_ctrl and obs…

…_http_middlebox tables (#100) * Add support for parsing ech related fields from TLS handshakes * Add support for parsing ooni_run_link_id xref: ooni/spec#297
ooni · Nov 20, 2024 · f4869da · f4869da
1 parent 825f5c7
commit f4869da
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 0 deletions.
diff --git a/oonidata/src/oonidata/models/dataformats.py b/oonidata/src/oonidata/models/dataformats.py
@@ -333,6 +333,8 @@ class TLSHandshake(BaseModel):
     peer_certificates: Optional[List[BinaryData]] = None
 
     server_name: Optional[str] = None
+    outer_server_name: Optional[str] = None
+    echconfig: Optional[str] = None
 
     tags: Optional[List[str]] = None
     tls_version: Optional[str] = None

diff --git a/oonidata/src/oonidata/models/observations.py b/oonidata/src/oonidata/models/observations.py
@@ -18,6 +18,7 @@ class MeasurementMeta:
     measurement_uid: str
     input: Optional[str]
     report_id: str
+    ooni_run_link_id: str
 
     measurement_start_time: datetime
 
@@ -114,6 +115,8 @@ class TLSObservation:
     failure: Failure
 
     server_name: str
+    outer_server_name: str
+    echconfig: str
     version: str
     cipher_suite: str
 
@@ -280,6 +283,8 @@ class WebObservation:
     tls_failure: Optional[Failure] = None
 
     tls_server_name: Optional[str] = None
+    tls_outer_server_name: Optional[str] = None
+    tls_echconfig: Optional[str] = None
     tls_version: Optional[str] = None
     tls_cipher_suite: Optional[str] = None
     tls_is_certificate_valid: Optional[bool] = None

diff --git a/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py
@@ -364,6 +364,8 @@ def measurement_to_tls_observation(
     tlso = TLSObservation(
         timestamp=make_timestamp(msmt_meta.measurement_start_time, tls_h.t),
         server_name=tls_h.server_name if tls_h.server_name else "",
+        outer_server_name=tls_h.outer_server_name if tls_h.outer_server_name else "",
+        echconfig=tls_h.echconfig if tls_h.echconfig else "",
         version=tls_h.tls_version if tls_h.tls_version else "",
         cipher_suite=tls_h.cipher_suite if tls_h.cipher_suite else "",
         end_entity_certificate_san_list=[],
@@ -714,10 +716,12 @@ def make_measurement_meta(msmt: BaseMeasurement, bucket_date: str) -> Measuremen
     if isinstance(input_, list):
         input_ = ":".join(input_)
 
+    annotations = msmt.annotations or {}
     return MeasurementMeta(
         measurement_uid=msmt.measurement_uid,
         report_id=msmt.report_id,
         input=input_,
+        ooni_run_link_id=str(annotations.get("ooni_run_link_id", "")),
         software_name=msmt.software_name,
         software_version=msmt.software_version,
         test_name=msmt.test_name,

diff --git a/oonipipeline/tests/test_ctrl.py b/oonipipeline/tests/test_ctrl.py
@@ -43,6 +43,7 @@
     test_name="web_connectivity",
     test_version="0.4.2",
     measurement_uid="TEST",
+    ooni_run_link_id="1234",
     input=None,
     report_id="TEST",
     measurement_start_time=datetime(2023, 10, 31, 15, 56, 12),