diff --git a/NDKVERSION b/NDKVERSION index 9dd96873ba..fef6a6fdf9 100644 --- a/NDKVERSION +++ b/NDKVERSION @@ -1 +1 @@ -26.0.10792818 +26.1.10909125 diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 25385a647e..503124aaec 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -78,7 +78,7 @@ func RunGroup(config RunGroupConfig) error { return err } if err := sess.MaybeLookupBackends(); err != nil { - log.WithError(err).Warn("Failed to discover OONI backends") + log.WithError(err).Errorf("Failed to discover OONI backends") return err } @@ -116,7 +116,9 @@ func RunGroup(config RunGroupConfig) error { ctl.RunType = config.RunType ctl.SetNettestIndex(i, len(group.Nettests)) if err = nt.Run(ctl); err != nil { - log.WithError(err).Errorf("Failed to run %s", group.Label) + // We used to emit an error here, now we emit a warning--the proper choice + // given that we continue running. See https://github.com/ooni/probe/issues/2576. + log.WithError(err).Warnf("Failed to run %s", group.Label) } } diff --git a/internal/dslx/dns.go b/internal/dslx/dns.go index 88d3bd65ec..62b86293f6 100644 --- a/internal/dslx/dns.go +++ b/internal/dslx/dns.go @@ -72,92 +72,117 @@ type ResolvedAddresses struct { // DNSLookupGetaddrinfo returns a function that resolves a domain name to // IP addresses using libc's getaddrinfo function. func DNSLookupGetaddrinfo(rt Runtime) Func[*DomainToResolve, *Maybe[*ResolvedAddresses]] { - return FuncAdapter[*DomainToResolve, *Maybe[*ResolvedAddresses]](func(ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { - // create trace - trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - rt.Logger(), - "[#%d] DNSLookup[getaddrinfo] %s", - trace.Index(), - input.Domain, - ) - - // setup - const timeout = 4 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // create the resolver - resolver := trace.NewStdlibResolver(rt.Logger()) - - // lookup - addrs, err := resolver.LookupHost(ctx, input.Domain) - - // stop the operation logger - ol.Stop(err) - - state := &ResolvedAddresses{ - Addresses: addrs, // maybe empty - Domain: input.Domain, - Trace: trace, - } - - return &Maybe[*ResolvedAddresses]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ResolveOperation, - State: state, - } - }) + return &dnsLookupGetaddrinfoFunc{rt} +} + +// dnsLookupGetaddrinfoFunc is the function returned by DNSLookupGetaddrinfo. +type dnsLookupGetaddrinfoFunc struct { + rt Runtime +} + +// Apply implements Func. +func (f *dnsLookupGetaddrinfoFunc) Apply( + ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { + + // create trace + trace := f.rt.NewTrace(f.rt.IDGenerator().Add(1), f.rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + f.rt.Logger(), + "[#%d] DNSLookup[getaddrinfo] %s", + trace.Index(), + input.Domain, + ) + + // setup + const timeout = 4 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // create the resolver + resolver := trace.NewStdlibResolver(f.rt.Logger()) + + // lookup + addrs, err := resolver.LookupHost(ctx, input.Domain) + + // stop the operation logger + ol.Stop(err) + + state := &ResolvedAddresses{ + Addresses: addrs, // maybe empty + Domain: input.Domain, + Trace: trace, + } + + return &Maybe[*ResolvedAddresses]{ + Error: err, + Observations: maybeTraceToObservations(trace), + Operation: netxlite.ResolveOperation, + State: state, + } } // DNSLookupUDP returns a function that resolves a domain name to // IP addresses using the given DNS-over-UDP resolver. -func DNSLookupUDP(rt Runtime, endpoint string) Func[*DomainToResolve, *Maybe[*ResolvedAddresses]] { - return FuncAdapter[*DomainToResolve, *Maybe[*ResolvedAddresses]](func(ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { - // create trace - trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - rt.Logger(), - "[#%d] DNSLookup[%s/udp] %s", - trace.Index(), - endpoint, - input.Domain, - ) - - // setup - const timeout = 4 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // create the resolver - resolver := trace.NewParallelUDPResolver( - rt.Logger(), - trace.NewDialerWithoutResolver(rt.Logger()), - endpoint, - ) - - // lookup - addrs, err := resolver.LookupHost(ctx, input.Domain) - - // stop the operation logger - ol.Stop(err) - - state := &ResolvedAddresses{ - Addresses: addrs, // maybe empty - Domain: input.Domain, - Trace: trace, - } - - return &Maybe[*ResolvedAddresses]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ResolveOperation, - State: state, - } - }) +func DNSLookupUDP(rt Runtime, resolver string) Func[*DomainToResolve, *Maybe[*ResolvedAddresses]] { + return &dnsLookupUDPFunc{ + Resolver: resolver, + rt: rt, + } +} + +// dnsLookupUDPFunc is the function returned by DNSLookupUDP. +type dnsLookupUDPFunc struct { + // Resolver is the MANDATORY endpointed of the resolver to use. + Resolver string + rt Runtime +} + +// Apply implements Func. +func (f *dnsLookupUDPFunc) Apply( + ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { + + // create trace + trace := f.rt.NewTrace(f.rt.IDGenerator().Add(1), f.rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + f.rt.Logger(), + "[#%d] DNSLookup[%s/udp] %s", + trace.Index(), + f.Resolver, + input.Domain, + ) + + // setup + const timeout = 4 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // create the resolver + resolver := trace.NewParallelUDPResolver( + f.rt.Logger(), + trace.NewDialerWithoutResolver(f.rt.Logger()), + f.Resolver, + ) + + // lookup + addrs, err := resolver.LookupHost(ctx, input.Domain) + + // stop the operation logger + ol.Stop(err) + + state := &ResolvedAddresses{ + Addresses: addrs, // maybe empty + Domain: input.Domain, + Trace: trace, + } + + return &Maybe[*ResolvedAddresses]{ + Error: err, + Observations: maybeTraceToObservations(trace), + Operation: netxlite.ResolveOperation, + State: state, + } } diff --git a/internal/dslx/dns_test.go b/internal/dslx/dns_test.go index 2ef3b82d3f..5dd6f79bd1 100644 --- a/internal/dslx/dns_test.go +++ b/internal/dslx/dns_test.go @@ -54,6 +54,13 @@ Test cases: - with success */ func TestGetaddrinfo(t *testing.T) { + t.Run("Get dnsLookupGetaddrinfoFunc", func(t *testing.T) { + f := DNSLookupGetaddrinfo(NewMinimalRuntime(model.DiscardLogger, time.Now())) + if _, ok := f.(*dnsLookupGetaddrinfoFunc); !ok { + t.Fatal("unexpected type, want dnsLookupGetaddrinfoFunc") + } + }) + t.Run("Apply dnsLookupGetaddrinfoFunc", func(t *testing.T) { domain := &DomainToResolve{ Domain: "example.com", @@ -61,9 +68,9 @@ func TestGetaddrinfo(t *testing.T) { } t.Run("with nil resolver", func(t *testing.T) { - f := DNSLookupGetaddrinfo( - NewMinimalRuntime(model.DiscardLogger, time.Now()), - ) + f := dnsLookupGetaddrinfoFunc{ + rt: NewMinimalRuntime(model.DiscardLogger, time.Now()), + } ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately cancel the lookup res := f.Apply(ctx, domain) @@ -77,8 +84,8 @@ func TestGetaddrinfo(t *testing.T) { t.Run("with lookup error", func(t *testing.T) { mockedErr := errors.New("mocked") - f := DNSLookupGetaddrinfo( - NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + f := dnsLookupGetaddrinfoFunc{ + rt: NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(&mocks.MeasuringNetwork{ MockNewStdlibResolver: func(logger model.DebugLogger) model.Resolver { return &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { @@ -87,7 +94,7 @@ func TestGetaddrinfo(t *testing.T) { } }, })), - ) + } res := f.Apply(context.Background(), domain) if res.Observations == nil || len(res.Observations) <= 0 { t.Fatal("unexpected empty observations") @@ -104,8 +111,8 @@ func TestGetaddrinfo(t *testing.T) { }) t.Run("with success", func(t *testing.T) { - f := DNSLookupGetaddrinfo( - NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + f := dnsLookupGetaddrinfoFunc{ + rt: NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ MockNewStdlibResolver: func(logger model.DebugLogger) model.Resolver { return &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { @@ -114,7 +121,7 @@ func TestGetaddrinfo(t *testing.T) { } }, })), - ) + } res := f.Apply(context.Background(), domain) if res.Observations == nil || len(res.Observations) <= 0 { t.Fatal("unexpected empty observations") @@ -144,6 +151,14 @@ Test cases: - with success */ func TestLookupUDP(t *testing.T) { + t.Run("Get dnsLookupUDPFunc", func(t *testing.T) { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + f := DNSLookupUDP(rt, "1.1.1.1:53") + if _, ok := f.(*dnsLookupUDPFunc); !ok { + t.Fatal("unexpected type, want dnsLookupUDPFunc") + } + }) + t.Run("Apply dnsLookupGetaddrinfoFunc", func(t *testing.T) { domain := &DomainToResolve{ Domain: "example.com", @@ -151,7 +166,7 @@ func TestLookupUDP(t *testing.T) { } t.Run("with nil resolver", func(t *testing.T) { - f := DNSLookupUDP(NewMinimalRuntime(model.DiscardLogger, time.Now()), "1.1.1.1:53") + f := dnsLookupUDPFunc{Resolver: "1.1.1.1:53", rt: NewMinimalRuntime(model.DiscardLogger, time.Now())} ctx, cancel := context.WithCancel(context.Background()) cancel() res := f.Apply(ctx, domain) @@ -165,8 +180,9 @@ func TestLookupUDP(t *testing.T) { t.Run("with lookup error", func(t *testing.T) { mockedErr := errors.New("mocked") - f := DNSLookupUDP( - NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + f := dnsLookupUDPFunc{ + Resolver: "1.1.1.1:53", + rt: NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(&mocks.MeasuringNetwork{ MockNewParallelUDPResolver: func(logger model.DebugLogger, dialer model.Dialer, endpoint string) model.Resolver { return &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { @@ -182,8 +198,7 @@ func TestLookupUDP(t *testing.T) { } }, })), - "1.1.1.1:53", - ) + } res := f.Apply(context.Background(), domain) if res.Observations == nil || len(res.Observations) <= 0 { t.Fatal("unexpected empty observations") @@ -200,8 +215,9 @@ func TestLookupUDP(t *testing.T) { }) t.Run("with success", func(t *testing.T) { - f := DNSLookupUDP( - NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + f := dnsLookupUDPFunc{ + Resolver: "1.1.1.1:53", + rt: NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ MockNewParallelUDPResolver: func(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { return &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { @@ -217,8 +233,7 @@ func TestLookupUDP(t *testing.T) { } }, })), - "1.1.1.1:53", - ) + } res := f.Apply(context.Background(), domain) if res.Observations == nil || len(res.Observations) <= 0 { t.Fatal("unexpected empty observations") diff --git a/internal/dslx/fxcore.go b/internal/dslx/fxcore.go index 329a8fc23c..64f95bbc0d 100644 --- a/internal/dslx/fxcore.go +++ b/internal/dslx/fxcore.go @@ -17,14 +17,6 @@ type Func[A, B any] interface { Apply(ctx context.Context, a A) B } -// FuncAdapter adapts a func to be a Func. -type FuncAdapter[A, B any] func(ctx context.Context, a A) B - -// Apply implements Func. -func (fa FuncAdapter[A, B]) Apply(ctx context.Context, a A) B { - return fa(ctx, a) -} - // Maybe is the result of an operation implemented by this package // that may fail such as [TCPConnect] or [TLSHandshake]. type Maybe[State any] struct { diff --git a/internal/dslx/tcp.go b/internal/dslx/tcp.go index d576b84fce..eaa54c2d30 100644 --- a/internal/dslx/tcp.go +++ b/internal/dslx/tcp.go @@ -15,50 +15,61 @@ import ( // TCPConnect returns a function that establishes TCP connections. func TCPConnect(rt Runtime) Func[*Endpoint, *Maybe[*TCPConnection]] { - return FuncAdapter[*Endpoint, *Maybe[*TCPConnection]](func(ctx context.Context, input *Endpoint) *Maybe[*TCPConnection] { - // create trace - trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - rt.Logger(), - "[#%d] TCPConnect %s", - trace.Index(), - input.Address, - ) - - // setup - const timeout = 15 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // obtain the dialer to use - dialer := trace.NewDialerWithoutResolver(rt.Logger()) - - // connect - conn, err := dialer.DialContext(ctx, "tcp", input.Address) - - // possibly register established conn for late close - rt.MaybeTrackConn(conn) - - // stop the operation logger - ol.Stop(err) - - state := &TCPConnection{ - Address: input.Address, - Conn: conn, // possibly nil - Domain: input.Domain, - Network: input.Network, - Trace: trace, - } - - return &Maybe[*TCPConnection]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ConnectOperation, - State: state, - } - }) + f := &tcpConnectFunc{rt} + return f +} + +// tcpConnectFunc is a function that establishes TCP connections. +type tcpConnectFunc struct { + rt Runtime +} + +// Apply applies the function to its arguments. +func (f *tcpConnectFunc) Apply( + ctx context.Context, input *Endpoint) *Maybe[*TCPConnection] { + + // create trace + trace := f.rt.NewTrace(f.rt.IDGenerator().Add(1), f.rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + f.rt.Logger(), + "[#%d] TCPConnect %s", + trace.Index(), + input.Address, + ) + + // setup + const timeout = 15 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // obtain the dialer to use + dialer := trace.NewDialerWithoutResolver(f.rt.Logger()) + + // connect + conn, err := dialer.DialContext(ctx, "tcp", input.Address) + + // possibly register established conn for late close + f.rt.MaybeTrackConn(conn) + + // stop the operation logger + ol.Stop(err) + + state := &TCPConnection{ + Address: input.Address, + Conn: conn, // possibly nil + Domain: input.Domain, + Network: input.Network, + Trace: trace, + } + + return &Maybe[*TCPConnection]{ + Error: err, + Observations: maybeTraceToObservations(trace), + Operation: netxlite.ConnectOperation, + State: state, + } } // TCPConnection is an established TCP connection. If you initialize diff --git a/internal/dslx/tcp_test.go b/internal/dslx/tcp_test.go index f5f1e28532..8748b634bb 100644 --- a/internal/dslx/tcp_test.go +++ b/internal/dslx/tcp_test.go @@ -13,6 +13,15 @@ import ( ) func TestTCPConnect(t *testing.T) { + t.Run("Get tcpConnectFunc", func(t *testing.T) { + f := TCPConnect( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + if _, ok := f.(*tcpConnectFunc); !ok { + t.Fatal("unexpected type. Expected: tcpConnectFunc") + } + }) + t.Run("Apply tcpConnectFunc", func(t *testing.T) { wasClosed := false plainConn := &mocks.Conn{ @@ -63,7 +72,7 @@ func TestTCPConnect(t *testing.T) { return tt.dialer }, })) - tcpConnect := TCPConnect(rt) + tcpConnect := &tcpConnectFunc{rt} endpoint := &Endpoint{ Address: "1.2.3.4:567", Network: "tcp", diff --git a/internal/registry/factory_test.go b/internal/registry/factory_test.go index b49ed21d56..49f11338a5 100644 --- a/internal/registry/factory_test.go +++ b/internal/registry/factory_test.go @@ -492,8 +492,10 @@ func TestNewFactory(t *testing.T) { inputPolicy: model.InputStrictlyRequired, }, "vanilla_tor": { - enabledByDefault: true, - inputPolicy: model.InputNone, + // The experiment crashes on Android and possibly also iOS. We want to + // control whether and when to run it using check-in. + //enabledByDefault: false, + inputPolicy: model.InputNone, }, "web_connectivity": { enabledByDefault: true, diff --git a/internal/registry/vanillator.go b/internal/registry/vanillator.go index cd8fae6583..fabf801662 100644 --- a/internal/registry/vanillator.go +++ b/internal/registry/vanillator.go @@ -16,8 +16,11 @@ func init() { *config.(*vanillator.Config), ) }, - config: &vanillator.Config{}, - enabledByDefault: true, + config: &vanillator.Config{}, + // We discussed this topic with @aanorbel. On Android this experiment crashes + // frequently because of https://github.com/ooni/probe/issues/2406. So, it seems + // more cautious to disable it by default and let the check-in API decide. + enabledByDefault: false, inputPolicy: model.InputNone, } }