convert more of v0.5's analysis to the ooni/data-like style

internal/pipeline/analysiscore.go

@@ -0,0 +1,79 @@
+package pipeline
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/optional"
+)
+
+// TODO(bassosimone): we are not extracting the data required to produce
+// duplicate-response based failure, which is also not implemented inside
+// of the v0.5 codebase, hence we're not regressing here.
+
+// Analysis aggregates the results of several analysis algorithms.
+//
+// Some values are optional. When a value IsNone, it means we could not run the
+// corresponding analysis algorithm, so we don't basically know.
+//
+// All the methods in this struct ARE NOT goroutine safe.
+//
+// The zero value of this struct is ready to use.
+type Analysis struct {
+	// DNSUnexpectedAddr lists all the DNS transaction IDs with unexpected addrs (i.e., IP
+	// addrs that have not have been also resolved by the TH).
+	DNSUnexpectedAddr []int64
+
+	// DNSUnexpectedAddrASN lists all the DNS transaction IDS with unexpected addrs ASNs.
+	DNSUnexpectedAddrASN []int64
+
+	// DNSUnexpectedBogon lists all the DNS transaction IDs for which we saw unexpected bogons.
+	DNSUnexpectedBogon []int64
+
+	// DNSUnexpectedFailure lists all the DNS transaction IDs containing unexpected DNS lookup failures.
+	DNSUnexpectedFailure []int64
+
+	// DNSWithTLSHandshakeFailureAddr lists all the DNS transaction IDs containing IP addresses
+	// for which the TH could perform a successful TLS handshake where the probe failed.
+	DNSWithTLSHandshakeFailureAddr []int64
+
+	// DNSExperimentFailure is a backward-compatibility value that contains the
+	// failure obtained when using getaddrinfo for the URL's domain
+	DNSExperimentFailure optional.Value[*string]
+
+	// HTTPExperimentFailure is a backward-compatibility value that contains the
+	// failure obtained for the final HTTP request made by the probe
+	HTTPExperimentFailure optional.Value[*string]
+
+	// HTTPUnexpectedFailure contains all the endpoint transaction IDs where
+	// the TH succeded while the probe failed to fetch a response
+	HTTPUnexpectedFailure []int64
+
+	// TCPUnexpectedFailure contains all the endpoint transaction IDs where the TH succeeded
+	// while the probe failed to connect (excluding obvious IPv6 issues).
+	TCPUnexpectedFailure []int64
+
+	// TLSUnexpectedFailure is like TCPUnexpectedFailure but for TLS.
+	TLSUnexpectedFailure []int64
+}
+
+// ComputeAll computes all the analysis flags using the DB.
+func (ax *Analysis) ComputeAll(db *DB) {
+	// DNS
+	ax.ComputeDNSExperimentFailure(db)
+	ax.ComputeDNSBogon(db)
+	ax.ComputeDNSUnexpectedFailure(db)
+	ax.ComputeDNSUnexpectedAddr(db)
+	ax.ComputeDNSUnexpectedAddrASN(db)
+	ax.ComputeDNSWithTLSHandshakeFailureAddr(db)
+
+	// TCP/IP
+	ax.ComputeTCPUnexpectedFailure(db)
+
+	// TLS
+	ax.ComputeTLSUnexpectedFailure(db)
+
+	// HTTP (core)
+	ax.ComputeHTTPUnexpectedFailure(db)
+	ax.ComputeHTTPExperimentFailure(db)
+
+	// HTTP (diff)
+
+}

internal/pipeline/analysisdns.go

@@ -0,0 +1,221 @@
+package pipeline
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/geoipx"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/optional"
+)
+
+// ComputeDNSExperimentFailure computes DNSExperimentFailure.
+func (ax *Analysis) ComputeDNSExperimentFailure(db *DB) {
+	for _, entry := range db.dnsByTxID {
+		// skip queries not for the original hostname
+		if db.urlHostname != entry.QueryHostname {
+			continue
+		}
+
+		// skip queries not using getaddrinfo
+		if dnsNormalizeEngineName(entry.Engine) != "getaddrinfo" {
+			continue
+		}
+
+		// skip successful cases
+		if entry.Failure == nil {
+			continue
+		}
+
+		// assign the first failure and return
+		ax.DNSExperimentFailure = optional.Some(entry.Failure)
+		return
+	}
+}
+
+// ComputeDNSUnexpectedBogon computes DNSUnexpectedBogon.
+func (ax *Analysis) ComputeDNSBogon(db *DB) {
+	for _, entry := range db.webByTxID {
+		// skip all the entries without a bogon
+		if !entry.IPAddressIsBogon {
+			continue
+		}
+
+		// skip cases where the TH also resolved the same bogon (e.g., as of 2023-11-23, the
+		// polito.it domain legitimately resolves to 192.168.59.6 and 192.168.40.1)
+		if entry.DNSLookupTHXref {
+			continue
+		}
+
+		// register the transaction containing the bogon.
+		ax.DNSUnexpectedBogon = append(ax.DNSUnexpectedBogon, entry.TransactionID)
+	}
+}
+
+// ComputeDNSUnexpectedFailure computes DNSUnexpectedFailure.
+func (ax *Analysis) ComputeDNSUnexpectedFailure(db *DB) {
+	// we cannot run this algorithm if the control failed or returned no IP addresses.
+	if db.thDNSFailure != nil {
+		return
+	}
+
+	// we cannot run this algorithm if the control returned no IP addresses.
+	if len(db.thDNSAddrs) <= 0 {
+		return
+	}
+
+	// inspect DNS lookup results
+	for _, entry := range db.dnsByTxID {
+		// skip cases without failures
+		if entry.Failure == nil {
+			continue
+		}
+
+		// skip cases that query the wrong domain name
+		if entry.QueryHostname != db.urlHostname {
+			continue
+		}
+
+		// A DoH failure is not information about the DNS blocking of the URL hostname
+		// we're measuring but rather about the DoH service being blocked.
+		//
+		// See https://github.com/ooni/probe/issues/2274
+		if entry.Engine == "doh" {
+			continue
+		}
+
+		// skip cases where there's no IPv6 addresses for a domain
+		if entry.QueryType == "AAAA" && *entry.Failure == netxlite.FailureDNSNoAnswer {
+			continue
+		}
+
+		// register the transaction as containing an unexpected DNS failure
+		ax.DNSUnexpectedFailure = append(ax.DNSUnexpectedFailure, entry.TransactionID)
+	}
+}
+
+func (ax *Analysis) dnsDiffHelper(db *DB, fx func(db *DB, entry *DNSObservation)) {
+	// we cannot run this algorithm if the control failed or returned no IP addresses.
+	if db.thDNSFailure != nil {
+		return
+	}
+
+	// we cannot run this algorithm if the control returned no IP addresses.
+	if len(db.thDNSAddrs) <= 0 {
+		return
+	}
+
+	// inspect DNS lookup results
+	for _, entry := range db.dnsByTxID {
+		// skip cases witht failures
+		if entry.Failure != nil {
+			continue
+		}
+
+		// skip cases that query the wrong domain name
+		if entry.QueryHostname != db.urlHostname {
+			continue
+		}
+
+		// Note: we include DoH-resolved addresses in this comparison
+		// because they should be ~as good as the TH addresses.
+
+		// invoke user defined function
+		fx(db, entry)
+	}
+}
+
+// ComputeDNSUnexpectedAddr computes DNSUnexpectedAddr.
+func (ax *Analysis) ComputeDNSUnexpectedAddr(db *DB) {
+	ax.dnsDiffHelper(db, func(db *DB, entry *DNSObservation) {
+		state := make(map[string]Origin)
+
+		for _, addr := range entry.IPAddrs {
+			state[addr] |= OriginProbe
+		}
+
+		for addr := range db.thDNSAddrs {
+			state[addr] |= OriginTH
+		}
+
+		for _, flags := range state {
+			if (flags & OriginTH) == 0 {
+				ax.DNSUnexpectedAddr = append(ax.DNSUnexpectedAddr, entry.TransactionID)
+				return
+			}
+		}
+	})
+}
+
+// ComputeDNSUnexpectedAddrASN computes DNSUnexpectedAddrASN.
+func (ax *Analysis) ComputeDNSUnexpectedAddrASN(db *DB) {
+	ax.dnsDiffHelper(db, func(db *DB, entry *DNSObservation) {
+		state := make(map[int64]Origin)
+
+		for _, addr := range entry.IPAddrs {
+			if asn, _, err := geoipx.LookupASN(addr); err == nil {
+				state[int64(asn)] |= OriginProbe
+			}
+		}
+
+		for addr := range db.thDNSAddrs {
+			if asn, _, err := geoipx.LookupASN(addr); err == nil {
+				state[int64(asn)] |= OriginTH
+			}
+		}
+
+		for _, flags := range state {
+			if (flags & OriginTH) == 0 {
+				ax.DNSUnexpectedAddrASN = append(ax.DNSUnexpectedAddrASN, entry.TransactionID)
+				return
+			}
+		}
+	})
+}
+
+// ComputeDNSWithTLSHandshakeFailureAddr computes DNSWithTLSHandshakeFailureAddr.
+func (ax *Analysis) ComputeDNSWithTLSHandshakeFailureAddr(db *DB) {
+	ax.dnsDiffHelper(db, func(db *DB, dns *DNSObservation) {
+		// walk through each resolved address in this DNS lookup
+		for _, addr := range dns.IPAddrs {
+
+			// find the corresponding endpoint measurement
+			for _, epnt := range db.webByTxID {
+
+				// skip entries related to a different address
+				if epnt.IPAddress != addr {
+					continue
+				}
+
+				// skip entries where we did not attempt a TLS handshake
+				if epnt.TLSHandshakeFailure.IsNone() {
+					continue
+				}
+
+				// skip entries where the handshake succeded
+				if epnt.TLSHandshakeFailure.Unwrap() == nil {
+					continue
+				}
+
+				// find the related TH measurement
+				thEpnt, good := db.thEpntByEpnt[epnt.Endpoint]
+
+				// skip cases where there's no TH entry
+				if !good {
+					continue
+				}
+
+				// skip cases where the TH did not perform an handshake (a bug?)
+				if thEpnt.TLSHandshakeFailure.IsNone() {
+					continue
+				}
+
+				// skip cases where the TH's handshake also failed
+				if thEpnt.TLSHandshakeFailure.Unwrap() != nil {
+					continue
+				}
+
+				// mark the DNS transaction as bad and stop
+				ax.DNSWithTLSHandshakeFailureAddr = append(ax.DNSWithTLSHandshakeFailureAddr, dns.TransactionID)
+				return
+			}
+		}
+	})
+}

internal/pipeline/analysishttpcore.go

@@ -0,0 +1,57 @@
+package pipeline
+
+import "github.com/ooni/probe-cli/v3/internal/optional"
+
+func (ax *Analysis) httpExperimentFailureHelper(db *DB, fx func(txId int64, probeFailure *string)) {
+	// skip if there's no final request
+	if db.webFinalRequest.IsNone() {
+		return
+	}
+	probeFR := db.webFinalRequest.Unwrap()
+
+	// skip if the HTTP failure is not defined (bug?)
+	if probeFR.HTTPFailure.IsNone() {
+		return
+	}
+
+	// skip if the final request succeded
+	// TODO(bassosimone): say that the probe succeeds and the TH fails, then what?
+	probeFailure := probeFR.HTTPFailure.Unwrap()
+	if probeFailure == nil {
+		return
+	}
+
+	// skip if the final request is not defined for the TH
+	if db.thWeb.IsNone() {
+		return
+	}
+	thFR := db.thWeb.Unwrap()
+
+	// skip if the failure is not defined for the TH
+	if thFR.HTTPFailure.IsNone() {
+		return
+	}
+
+	// skip if also the TH's HTTP request failed
+	thFailure := thFR.HTTPFailure.Unwrap()
+	if thFailure != nil {
+		return
+	}
+
+	// invoke user defined func
+	fx(probeFR.TransactionID, probeFailure)
+}
+
+// ComputeHTTPUnexpectedFailure computes HTTPUnexpectedFailure.
+func (ax *Analysis) ComputeHTTPUnexpectedFailure(db *DB) {
+	ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure *string) {
+		ax.HTTPUnexpectedFailure = append(ax.HTTPUnexpectedFailure, txId)
+	})
+}
+
+// ComputeHTTPExperimentFailure computes HTTPExperimentFailure.
+func (ax *Analysis) ComputeHTTPExperimentFailure(db *DB) {
+	ax.httpExperimentFailureHelper(db, func(txId int64, probeFailure *string) {
+		ax.HTTPExperimentFailure = optional.Some(probeFailure)
+	})
+}

internal/pipeline/analysishttpdiff.go

@@ -0,0 +1 @@
+package pipeline