feat: add openvpn experiment

TEST

@@ -0,0 +1,6 @@
+#!/bin/sh
+OONI_FORCE_ENABLE_EXPERIMENT=1 ./miniooni openvpn \
+    -O SafeCA=base64:LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJZakNDQVFpZ0F3SUJBZ0lCQVRBS0JnZ3Foa2pPUFFRREFqQVhNUlV3RXdZRFZRUURFd3hNUlVGUUlGSnYKYjNRZ1EwRXdIaGNOTWpFeE1UQXlNVGt3TlRNM1doY05Nall4TVRBeU1Ua3hNRE0zV2pBWE1SVXdFd1lEVlFRRApFd3hNUlVGUUlGSnZiM1FnUTBFd1dUQVRCZ2NxaGtqT1BRSUJCZ2dxaGtqT1BRTUJCd05DQUFReE9YQkd1K2dmCnBqSHpWdGVHVFdMNlhuRnh0RW5LTUZwS2FKa0EvVk9IbUVTem9Mc1pSUXh0ODhHc3N4YXFDMDFKMTdpZFFpcXYKemdOcGVkbXR2RnR5bzBVd1F6QU9CZ05WSFE4QkFmOEVCQU1DQXFRd0VnWURWUjBUQVFIL0JBZ3dCZ0VCL3dJQgpBVEFkQmdOVkhRNEVGZ1FVWmRvVWxKckNJVU5GcnBmZkFxK0xRam53RXo0d0NnWUlLb1pJemowRUF3SURTQUF3ClJRSWdmcjN3NHRuUkcrTmRJM0xzR1Bsc1JrdEdLMjB4SFR6c0Izb3JCMHlDNmNJQ0lRQ0IrLzl5OG5tU1N0Zk4KVlVNVXlrMmhOZDcva0M4bkwyMjJUVEQ3VlpVdHNnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== \
+    -O SafeCert=base64:LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNkekNDQWgyZ0F3SUJBZ0lRQy9QRUlmdDBZTGlmRTFIZU8rYjZYakFLQmdncWhrak9QUVFEQWpBek1URXcKTHdZRFZRUUREQ2hNUlVGUUlGSnZiM1FnUTBFZ0tHTnNhV1Z1ZENCalpYSjBhV1pwWTJGMFpYTWdiMjVzZVNFcApNQjRYRFRJME1ETXlOakU0TlRJME1Gb1hEVEkwTURRek1ERTROVEkwTUZvd0ZERVNNQkFHQTFVRUF4TUpWVTVNClNVMUpWRVZFTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFvWE81azZNOVE4UjMKS0dHU0Q2Y052YktCck9Kc1pOQ1dFMVlVK3dOQ2JERGJPSjlaQmNmSWc4d2wrODNWeGkySUtaL0pNSTJaUXQzegp4UTdRa1NCY0hEK2lZdWlxOVhQL283dDZMTHhzY3FoRU1LWW41UFEwaWo2Qnc5RFUyOXA4c2lXU0lrNUprY3dYCm5rVmVQTUNvb0tMN2NGZ3lyanQrM3liM3Z1TmI3TnZDM3V3akUyR00wa2FLVGp2cEN5MDUyUzI4ZGo4NCtUTksKSHJYa1JzRGNuTmQrZ2xrM2hSZ005VzNCak9uczN5eitnV2pjT3lraXdITElnd05CWjE3aVltNEpUZnV3dnY2RQpJREJQZEVtSUsxYWovWUlRSWNpekxCODFBTXk2U0l0cytVc0V3MkZyVW9zWlRSMUJiZUMyQVNCVEhBOTY0RzBqCmJmalU5V1pScFFJREFRQUJvMmN3WlRBT0JnTlZIUThCQWY4RUJBTUNCNEF3RXdZRFZSMGxCQXd3Q2dZSUt3WUIKQlFVSEF3SXdIUVlEVlIwT0JCWUVGSEJ4a0xvZVpCU29UbkpORmJmUjNoSGlrbi93TUI4R0ExVWRJd1FZTUJhQQpGSDFLWXRqL0swbkVlYmFpc2R5T1BtTUhYS2orTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSUJqNDRiQTRieURECklzT1VQTHVLZEU5Skg4alFkYnZvZTdITUI1dU1IWlZtQWlFQTBrWXM0aUVoRGZOWEx5Y0NxeE5qd0xDYUxMNmwKR055aHZOenNkd2pGRzlVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t \
+    -O SafeKey=base64:LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBb1hPNWs2TTlROFIzS0dHU0Q2Y052YktCck9Kc1pOQ1dFMVlVK3dOQ2JERGJPSjlaCkJjZklnOHdsKzgzVnhpMklLWi9KTUkyWlF0M3p4UTdRa1NCY0hEK2lZdWlxOVhQL283dDZMTHhzY3FoRU1LWW4KNVBRMGlqNkJ3OURVMjlwOHNpV1NJazVKa2N3WG5rVmVQTUNvb0tMN2NGZ3lyanQrM3liM3Z1TmI3TnZDM3V3agpFMkdNMGthS1RqdnBDeTA1MlMyOGRqODQrVE5LSHJYa1JzRGNuTmQrZ2xrM2hSZ005VzNCak9uczN5eitnV2pjCk95a2l3SExJZ3dOQloxN2lZbTRKVGZ1d3Z2NkVJREJQZEVtSUsxYWovWUlRSWNpekxCODFBTXk2U0l0cytVc0UKdzJGclVvc1pUUjFCYmVDMkFTQlRIQTk2NEcwamJmalU5V1pScFFJREFRQUJBb0lCQUdpZEZMVXVROGRDYVRkWQpLWTFNNEdGM1pnRUE0ZDFkTHJFdXlQOXd1RHhrVjlmVG9KcFhQbnp2N2ZqQUFmR2NsU3JyWnpDM3Y0UU5UeVB6Cm1uOCs2WkJBUjFjeGpYem9BZEEwN1ZCSmN6ZkVBaE5IeG5mYktLUWZKblFjUDZDSmJOejk3VGVmWkpvOUZWeXYKZEFXSGpId3h2eHluZVlkZDg3SmlqSXM2eE5pdThZN1ZMSGppU0IvdkUvNnA3YlBYbkZ4L0E3WVl0SFdkallhYwplNGtuRFR4czEwUzVYTEI4V3hvZnNqK1ZGUklkUkFjUURZYnZpTlZ3cXlEVXV3TFFJZ3Nlc1lldUEyVmF3ZU5XClpVTXN2TGZjdXB2QUU2b2ZGM21XcXh2Unljd0ZZRjB4ZUVPNzNrQ0RJbTM5cHdDdVNQOFlYSmk4U3VHUDVQZmwKUTZBQlNBMENnWUVBeDZSQlFHdjM4TE9UOVkzdVV2am9hanNKY25QZWR3ejlITGJjOGJhUk0yMGdJaUdUZlR0Kwp5RnNhT2Raa05GSmtCd3A4amhwaTcvRzBaWURhbUZ1bkhBY2JyUFlhV2FodXhWRlp6OStyOWlrQmtFVzEwOFJ2CkFpNnhXTklNcXlaRkhoZ2d2MGhzeWZFbmoxeSs5TGR4UWVncHlteDhBQzRpSVBtV0cvak9QODhDZ1lFQXp3ZVQKNFBpK2sweUdaSU5DVGdDSmY1blVRT2xnRXF3bGtXbE9hVzNpODdVdnlDaDJZbG5NQjhkSnUwajZBYkZHQWtKMAo0NlFKekIvWkdIZ1lXU0tDdjhLUk9sdkExeElpdXdvaDU3bXcxUUFKNzE2SktRZXk4SXRWUDJEb2JhQXFxTHlpCktkQjJnNjZJZTFaUTAzV2pyaWZZWUE4cUlLeUFRQU5LS00wMFlFc0NnWUJHNjJtbXFmUWxGSlgrQ0JKZWRUK1MKNVRBQThYcFl6a3RvRk9tK0QvM2F4K3cvVTdBaUw2MWxIVC9leGZOSXh1L3p0RnowMmhqRlpoYVFiRXE3RHV2NQpQK2tyOHl6L0pwOWJCd0Fob1RKa09zTHNibWNlT0V5NitMMVZjU0RBOTlKYjAzUm1ueUxPUmhXb2p1amk0L3VlCnp3dHhka0pDaWlEamwrWWtQNmw4N3dLQmdEQWhUdHZLZUdPK01yQkRZN0xHcFRDcERwTllyaUVwTEVLMS9LaSsKQnprcE1rYVNRWk56MU44cUVaWWN6U0ovbUFzR2NDNU1BSXNZREZ5SDd3RXA2TU96OUJkaEpWL2FzNEJRUnJtYwpqZU8yOHBoWG5nT1A2cVhKZnJRUlQzZk0rSjNwM0xsajRXbVR0ZDhXbmd2TEdaWnZaUHBRRERjMFkweCtkMHN0CkNiUXBBb0dCQUkwWjAxU2w3b2wvdTlxNVhrTUhtNUR3UkVIRDhPOXZYZHR6NEQyaUpUK2JjMStPT3ZHTGQzS04KRHBJbVpJYUFUcFh1OEZxdXVDUzN4WDRDNjJSSmgzbC9IRXArWmQ4V2YxL0UxUUFBSHNBSGk0elM3R2RSSnJOdAo3cUZ4UkVsaVJ0Wm10a09TL1B3YzY5eTB0aENJRCtIazhqWGlQRnBEejNMNmNZSWJTaUJqCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== \
+-n

internal/engine/inputloader.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 
 	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/experiment/openvpn"
 	"github.com/ooni/probe-cli/v3/internal/fsx"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/registry"
@@ -29,6 +30,8 @@ var (
 type InputLoaderSession interface {
 	CheckIn(ctx context.Context,
 		config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error)
+	FetchOpenVPNConfig(ctx context.Context,
+		provider, cc string) (*model.OOAPIVPNProviderConfig, error)
 }
 
 // InputLoaderLogger is the logger according to an InputLoader.
@@ -300,6 +303,16 @@ func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]mode
 
 // loadRemote loads inputs from a remote source.
 func (il *InputLoader) loadRemote(ctx context.Context) ([]model.OOAPIURLInfo, error) {
+	switch registry.CanonicalizeExperimentName(il.ExperimentName) {
+	case "openvpn":
+		return il.loadRemoteOpenVPN(ctx)
+	default:
+		return il.loadRemoteWebConnectivity(ctx)
+	}
+}
+
+// loadRemoteWebConnectivity loads webconnectivity inputs from a remote source.
+func (il *InputLoader) loadRemoteWebConnectivity(ctx context.Context) ([]model.OOAPIURLInfo, error) {
 	config := il.CheckInConfig
 	if config == nil {
 		// Note: Session.CheckIn documentation says it will fill in
@@ -318,6 +331,39 @@ func (il *InputLoader) loadRemote(ctx context.Context) ([]model.OOAPIURLInfo, er
 	return reply.WebConnectivity.URLs, nil
 }
 
+// loadRemoteOpenVPN loads openvpn inputs from a remote source.
+func (il *InputLoader) loadRemoteOpenVPN(ctx context.Context) ([]model.OOAPIURLInfo, error) {
+	// VPN Inputs do not match exactly the semantics expected from [model.OOAPIURLInfo],
+	// since OOAPIURLInfo is oriented towards webconnectivity,
+	// but we force VPN targets in the URL and ignore all the other fields.
+	urls := make([]model.OOAPIURLInfo, 0)
+
+	// The openvpn experiment contains an array of the providers that the API knows about.
+	for _, provider := range openvpn.APIEnabledProviders {
+		reply, err := il.vpnConfig(ctx, provider)
+		if err != nil {
+			break
+		}
+		// here we're just collecting all the inputs. we also cache the configs so that
+		// each experiment run can access the credentials for a given provider.
+		for _, input := range reply.Inputs {
+			urls = append(urls, model.OOAPIURLInfo{URL: input})
+		}
+	}
+
+	if len(urls) == 0 {
+		// loadRemote returns ErrNoURLsReturned at this point for webconnectivity,
+		// but for OpenVPN we want to return a sensible default to be
+		// able to probe some endpoints even in very restrictive environments.
+		// Do note this means that you have to provide valid credentials
+		// by some other means.
+		for _, endpoint := range openvpn.DefaultEndpoints {
+			urls = append(urls, model.OOAPIURLInfo{URL: endpoint.AsInputURI()})
+		}
+	}
+	return urls, nil
+}
+
 // checkIn executes the check-in and filters the returned URLs to exclude
 // the URLs that are not part of the requested categories. This is done for
 // robustness, just in case we or the API do something wrong.
@@ -336,6 +382,15 @@ func (il *InputLoader) checkIn(
 	return reply, nil
 }
 
+// vpnConfig fetches vpn information for the configured providers
+func (il *InputLoader) vpnConfig(ctx context.Context, provider string) (*model.OOAPIVPNProviderConfig, error) {
+	reply, err := il.Session.FetchOpenVPNConfig(ctx, provider, "XX")
+	if err != nil {
+		return nil, err
+	}
+	return reply, nil
+}
+
 // preventMistakes makes the code more robust with respect to any possible
 // integration issue where the backend returns to us URLs that don't
 // belong to the category codes we requested.

internal/engine/inputloader_test.go

@@ -444,9 +444,13 @@ func TestInputLoaderReadfileScannerFailure(t *testing.T) {
 // InputLoaderMockableSession is a mockable session
 // used by InputLoader tests.
 type InputLoaderMockableSession struct {
-	// Output contains the output of CheckIn. It should
-	// be nil when Error is not-nil.
-	Output *model.OOAPICheckInResultNettests
+	// CheckinOutput contains the output of CheckIn. It should
+	// be nil when Error is non-nil.
+	CheckinOutput *model.OOAPICheckInResultNettests
+
+	// FetchOpenVPNConfigOutput contains the output of FetchOpenVPNConfig.
+	// It should be nil when Error is non-nil.
+	FetchOpenVPNConfigOutput *model.OOAPIVPNProviderConfig
 
 	// Error is the error to be returned by CheckIn. It
 	// should be nil when Output is not-nil.
@@ -456,10 +460,19 @@ type InputLoaderMockableSession struct {
 // CheckIn implements InputLoaderSession.CheckIn.
 func (sess *InputLoaderMockableSession) CheckIn(
 	ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) {
-	if sess.Output == nil && sess.Error == nil {
+	if sess.CheckinOutput == nil && sess.Error == nil {
+		return nil, errors.New("both Output and Error are nil")
+	}
+	return sess.CheckinOutput, sess.Error
+}
+
+// FetchOpenVPNConfig implements InputLoaderSession.FetchOpenVPNConfig.
+func (sess *InputLoaderMockableSession) FetchOpenVPNConfig(
+	ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) {
+	if sess.FetchOpenVPNConfigOutput == nil && sess.Error == nil {
 		return nil, errors.New("both Output and Error are nil")
 	}
-	return sess.Output, sess.Error
+	return sess.FetchOpenVPNConfigOutput, sess.Error
 }
 
 func TestInputLoaderCheckInFailure(t *testing.T) {
@@ -480,7 +493,7 @@ func TestInputLoaderCheckInFailure(t *testing.T) {
 func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) {
 	il := &InputLoader{
 		Session: &InputLoaderMockableSession{
-			Output: &model.OOAPICheckInResultNettests{},
+			CheckinOutput: &model.OOAPICheckInResultNettests{},
 		},
 	}
 	out, err := il.loadRemote(context.Background())
@@ -495,7 +508,7 @@ func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) {
 func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) {
 	il := &InputLoader{
 		Session: &InputLoaderMockableSession{
-			Output: &model.OOAPICheckInResultNettests{
+			CheckinOutput: &model.OOAPICheckInResultNettests{
 				WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{},
 			},
 		},
@@ -521,7 +534,7 @@ func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) {
 	}}
 	il := &InputLoader{
 		Session: &InputLoaderMockableSession{
-			Output: &model.OOAPICheckInResultNettests{
+			CheckinOutput: &model.OOAPICheckInResultNettests{
 				WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{
 					URLs: expect,
 				},

internal/engine/session.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"sync"
@@ -67,6 +66,7 @@ type Session struct {
 	softwareName             string
 	softwareVersion          string
 	tempDir                  string
+	vpnConfig                map[string]model.OOAPIVPNProviderConfig
 
 	// closeOnce allows us to call Close just once.
 	closeOnce sync.Once
@@ -155,7 +155,7 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) {
 	// use the temporary directory on the current system. This should
 	// work on Desktop. We tested that it did also work on iOS, but
 	// we have also seen on 2020-06-10 that it does not work on Android.
-	tempDir, err := ioutil.TempDir(config.TempDir, "ooniengine")
+	tempDir, err := os.MkdirTemp(config.TempDir, "ooniengine")
 	if err != nil {
 		return nil, err
 	}
@@ -178,6 +178,7 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) {
 		torArgs:                 config.TorArgs,
 		torBinary:               config.TorBinary,
 		tunnelDir:               config.TunnelDir,
+		vpnConfig:               make(map[string]model.OOAPIVPNProviderConfig),
 	}
 	proxyURL := config.ProxyURL
 	if proxyURL != nil {
@@ -374,6 +375,25 @@ func (s *Session) FetchTorTargets(
 	return clnt.FetchTorTargets(ctx, cc)
 }
 
+// FetchOpenVPNConfig fetches openvpn config from the API if it's not found in the
+// internal cache. We do this to avoid hitting the API for every input.
+func (s *Session) FetchOpenVPNConfig(
+	ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) {
+	if config, ok := s.vpnConfig[provider]; ok {
+		return &config, nil
+	}
+	clnt, err := s.NewOrchestraClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	config, err := clnt.FetchOpenVPNConfig(ctx, provider, cc)
+	if err != nil {
+		return nil, err
+	}
+	s.vpnConfig[provider] = config
+	return &config, nil
+}
+
 // KeyValueStore returns the configured key-value store.
 func (s *Session) KeyValueStore() model.KeyValueStore {
 	return s.kvStore

internal/engine/session_internal_test.go

@@ -253,6 +253,19 @@ func TestSessionMaybeLookupLocationContextLookupLocationContextFailure(t *testin
 	}
 }
 
+func TestSessionFetchOpenVPNConfigWithCancelledContext(t *testing.T) {
+	sess := &Session{}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // cause failure
+	resp, err := sess.FetchOpenVPNConfig(ctx, "riseup", "XX")
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected", err)
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
 func TestSessionFetchTorTargetsWithCancelledContext(t *testing.T) {
 	sess := &Session{}
 	ctx, cancel := context.WithCancel(context.Background())