From 0f1771bd8e2b6eeb47571a4e274bca127a8fd6df Mon Sep 17 00:00:00 2001 From: bahrimootaz <105734428+bahrimootaz@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:44:56 +0100 Subject: [PATCH] Add Support for Parsing .xctestrun file and use it to start xctest (#525) This pull request introduces enhancements to the go-ios tool by adding the ability to parse .xctestrun files, enabling seamless execution of XCTest. The key updates include: Key updates include: 1. Parsing .xctestrun File: A new parseFile method has been implemented to extract test configuration details from the .xctestrun file. The parsed data includes all details needed to execute an xctest like TestHostBundleIdentifier, TestBundlePath, OnlyTestIdentifiers, SkipTestIdentifiers, IsUITestBundle, CommandLineArguments, EnvironmentVariables, and TestingEnvironmentVariables of the xctestrun file. Added a validation step to ensure the FormatVersion of the .xctestrun file is supported. Currently, only FormatVersion 1 is accepted, with clear error messages for unsupported versions. 2. Enhanced CLI Command: Introduced a new ios runxctest command in the CLI with updated explanations for better usability. Allows users to specify xctestrun file path and output preferences. 3. Improved Test Coverage: Added test cases to validate parsing of .xctestrun file. Ensured robust assertions for critical fields like TestHostBundleIdentifier and TestBundlePath. These updates simplify and enhance the process of running XCTest by directly utilizing .xctestrun files, reducing manual setup and ensuring better adaptability to diverse testing scenarios. --- ios/testmanagerd/xctestrunnerutils_test.go | 304 +++++++++++++++++++++ ios/testmanagerd/xctestrunutils.go | 172 ++++++++++++ ios/testmanagerd/xcuitestrunner.go | 27 +- main.go | 35 +++ 4 files changed, 534 insertions(+), 4 deletions(-) create mode 100644 ios/testmanagerd/xctestrunnerutils_test.go create mode 100755 ios/testmanagerd/xctestrunutils.go diff --git a/ios/testmanagerd/xctestrunnerutils_test.go b/ios/testmanagerd/xctestrunnerutils_test.go new file mode 100644 index 00000000..a6a330b3 --- /dev/null +++ b/ios/testmanagerd/xctestrunnerutils_test.go @@ -0,0 +1,304 @@ +package testmanagerd + +import ( + "github.com/danielpaulus/go-ios/ios" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Helper function to create mock data and parse the .xctestrun file +func createAndParseXCTestRunFile(t *testing.T) xCTestRunData { + // Arrange: Create a temporary .xctestrun file with mock data + tempFile, err := os.CreateTemp("", "testfile*.xctestrun") + assert.NoError(t, err, "Failed to create temp file") + defer os.Remove(tempFile.Name()) // Cleanup after test + + xcTestRunFileFormatVersion1 := ` + + + + + RunnerTests + + BlueprintName + RunnerTests + BlueprintProviderName + Runner + BlueprintProviderRelativePath + Runner.xcodeproj + BundleIdentifiersForCrashReportEmphasis + + com.example.myApp + com.example.myApp.RunnerTests + + CommandLineArguments + + DefaultTestExecutionTimeAllowance + 600 + DependentProductPaths + + __TESTROOT__/Release-iphoneos/Runner.app + __TESTROOT__/Release-iphoneos/Runner.app/PlugIns/RunnerTests.xctest + + DiagnosticCollectionPolicy + 1 + EnvironmentVariables + + APP_DISTRIBUTOR_ID_OVERRIDE + com.apple.AppStore + OS_ACTIVITY_DT_MODE + YES + SQLITE_ENABLE_THREAD_ASSERTIONS + 1 + TERM + dumb + + IsAppHostedTestBundle + + ParallelizationEnabled + + PreferredScreenCaptureFormat + screenRecording + ProductModuleName + RunnerTests + RunOrder + 0 + SystemAttachmentLifetime + deleteOnSuccess + TestBundlePath + __TESTHOST__/PlugIns/RunnerTests.xctest + TestHostBundleIdentifier + com.example.myApp + TestHostPath + __TESTROOT__/Release-iphoneos/Runner.app + TestLanguage + + TestRegion + + TestTimeoutsEnabled + + TestingEnvironmentVariables + + DYLD_INSERT_LIBRARIES + __TESTHOST__/Frameworks/libXCTestBundleInject.dylib + XCInjectBundleInto + unused + Test + xyz + + ToolchainsSettingValue + + UserAttachmentLifetime + deleteOnSuccess + OnlyTestIdentifiers + + TestClass1/testMethod1 + TestClass2/testMethod1 + + SkipTestIdentifiers + + TestClass1/testMethod2 + TestClass2/testMethod2 + + IsUITestBundle + + + __xctestrun_metadata__ + + ContainerInfo + + ContainerName + Runner + SchemeName + Runner + + FormatVersion + 1 + + + + ` + _, err = tempFile.WriteString(xcTestRunFileFormatVersion1) + assert.NoError(t, err, "Failed to write mock data to temp file") + tempFile.Close() + + // Act: Use the codec to parse the temp file + xcTestRunData, err := parseFile(tempFile.Name()) + + // Assert: Verify the parsed data + assert.NoError(t, err, "Failed to parse .xctestrun file") + assert.NotNil(t, xcTestRunData, "Parsed data should not be nil") + + return xcTestRunData +} + +func TestTestHostBundleIdentifier(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, "com.example.myApp", xcTestRunData.TestConfig.TestHostBundleIdentifier, "TestHostBundleIdentifier mismatch") +} + +func TestTestBundlePath(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, "__TESTHOST__/PlugIns/RunnerTests.xctest", xcTestRunData.TestConfig.TestBundlePath, "TestBundlePath mismatch") +} + +func TestEnvironmentVariables(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, map[string]any{ + "APP_DISTRIBUTOR_ID_OVERRIDE": "com.apple.AppStore", + "OS_ACTIVITY_DT_MODE": "YES", + "SQLITE_ENABLE_THREAD_ASSERTIONS": "1", + "TERM": "dumb", + }, xcTestRunData.TestConfig.EnvironmentVariables, "EnvironmentVariables mismatch") +} + +func TestTestingEnvironmentVariables(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, map[string]any{ + "DYLD_INSERT_LIBRARIES": "__TESTHOST__/Frameworks/libXCTestBundleInject.dylib", + "XCInjectBundleInto": "unused", + "Test": "xyz", + }, xcTestRunData.TestConfig.TestingEnvironmentVariables, "TestingEnvironmentVariables mismatch") +} + +func TestCommandLineArguments(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, []string{}, xcTestRunData.TestConfig.CommandLineArguments, "CommandLineArguments mismatch") +} + +func TestOnlyTestIdentifiers(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, []string{ + "TestClass1/testMethod1", + "TestClass2/testMethod1", + }, xcTestRunData.TestConfig.OnlyTestIdentifiers, "OnlyTestIdentifiers mismatch") +} + +func TestSkipTestIdentifiers(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, []string{ + "TestClass1/testMethod2", + "TestClass2/testMethod2", + }, xcTestRunData.TestConfig.SkipTestIdentifiers, "SkipTestIdentifiers mismatch") +} + +func TestFormatVersion(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, 1, xcTestRunData.XCTestRunMetadata.FormatVersion, "FormatVersion mismatch") +} + +func TestIsUITestBundle(t *testing.T) { + xcTestRunData := createAndParseXCTestRunFile(t) + assert.Equal(t, true, xcTestRunData.TestConfig.IsUITestBundle, "IsUITestBundle mismatch") +} + +func TestParseXCTestRunNotSupportedForFormatVersionOtherThanOne(t *testing.T) { + // Arrange: Create a temporary .xctestrun file with mock data + tempFile, err := os.CreateTemp("", "testfile*.xctestrun") + assert.NoError(t, err, "Failed to create temp file") + defer os.Remove(tempFile.Name()) // Cleanup after test + + xcTestRunFileFormatVersion2 := ` + + + + + __xctestrun_metadata__ + + FormatVersion + 2 + + + + ` + _, err = tempFile.WriteString(xcTestRunFileFormatVersion2) + assert.NoError(t, err, "Failed to write mock data to temp file") + tempFile.Close() + + // Act: Use the codec to parse the temp file + _, err = parseFile(tempFile.Name()) + + // Assert the Error Message + assert.Equal(t, "go-ios currently only supports .xctestrun files in formatVersion 1: The formatVersion of your xctestrun file is 2, feel free to open an issue in https://github.com/danielpaulus/go-ios/issues to add support", err.Error(), "Error Message mismatch") +} + +// Helper function to create testConfig from parsed mock data +func createTestConfigFromParsedMockData(t *testing.T) (TestConfig, ios.DeviceEntry, *TestListener) { + // Arrange: Create parsed XCTestRunData using the helper function + xcTestRunData := createAndParseXCTestRunFile(t) + + // Mock dependencies + mockDevice := ios.DeviceEntry{ + DeviceID: 8110, + } + mockListener := &TestListener{} + + // Act: Convert XCTestRunData to TestConfig + testConfig, err := xcTestRunData.buildTestConfig(mockDevice, mockListener) + + // Assert: Validate the returned TestConfig + assert.NoError(t, err, "Error converting to TestConfig") + + return testConfig, mockDevice, mockListener +} + +func TestConfigTestRunnerBundleId(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, "com.example.myApp", testConfig.TestRunnerBundleId, "TestRunnerBundleId mismatch") +} + +func TestConfigXctestConfigName(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, "RunnerTests.xctest", testConfig.XctestConfigName, "XctestConfigName mismatch") +} + +func TestConfigCommandLineArguments(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, []string{}, testConfig.Args, "data mismatch") +} + +func TestConfigEnvironmentVariables(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, map[string]any{ + "APP_DISTRIBUTOR_ID_OVERRIDE": "com.apple.AppStore", + "OS_ACTIVITY_DT_MODE": "YES", + "SQLITE_ENABLE_THREAD_ASSERTIONS": "1", + "TERM": "dumb", + "DYLD_INSERT_LIBRARIES": "__TESTHOST__/Frameworks/libXCTestBundleInject.dylib", + "XCInjectBundleInto": "unused", + "Test": "xyz", + }, testConfig.Env, "EnvironmentVariables mismatch") +} + +func TestConfigTestsToRun(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, []string{ + "TestClass1/testMethod1", + "TestClass2/testMethod1", + }, testConfig.TestsToRun, "TestsToRun mismatch") +} + +func TestConfigTestsToSkip(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, []string{ + "TestClass1/testMethod2", + "TestClass2/testMethod2", + }, testConfig.TestsToSkip, "TestsToSkip mismatch") +} + +func TestConfigXcTest(t *testing.T) { + testConfig, _, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, false, testConfig.XcTest, "XcTest mismatch") +} + +func TestConfigDevice(t *testing.T) { + testConfig, mockDevice, _ := createTestConfigFromParsedMockData(t) + assert.Equal(t, mockDevice, testConfig.Device, "Device mismatch") +} + +func TestConfigListener(t *testing.T) { + testConfig, _, mockListener := createTestConfigFromParsedMockData(t) + assert.Equal(t, mockListener, testConfig.Listener, "Listener mismatch") +} diff --git a/ios/testmanagerd/xctestrunutils.go b/ios/testmanagerd/xctestrunutils.go new file mode 100755 index 00000000..7dca8b07 --- /dev/null +++ b/ios/testmanagerd/xctestrunutils.go @@ -0,0 +1,172 @@ +package testmanagerd + +import ( + "bytes" + "errors" + "fmt" + "github.com/danielpaulus/go-ios/ios" + "howett.net/plist" + "io" + "maps" + "os" + "path/filepath" +) + +// xctestrunutils provides utilities for parsing `.xctestrun` files with FormatVersion 1. +// It simplifies the extraction of test configurations and metadata into structured objects (`xCTestRunData`), +// enabling efficient setup for iOS test execution. +// +// Features: +// - Parses `.xctestrun` files to extract test metadata and configurations. +// - Supports building `TestConfig` objects for test execution. +// +// Note: Only `.xctestrun` files with `FormatVersion` 1 are supported. For other versions, +// contributions or requests for support can be made in the relevant GitHub repository. + +// xCTestRunData represents the structure of an .xctestrun file +type xCTestRunData struct { + TestConfig schemeData `plist:"-"` + XCTestRunMetadata xCTestRunMetadata `plist:"__xctestrun_metadata__"` +} + +// schemeData represents the structure of a scheme-specific test configuration +type schemeData struct { + TestHostBundleIdentifier string + TestBundlePath string + SkipTestIdentifiers []string + OnlyTestIdentifiers []string + IsUITestBundle bool + CommandLineArguments []string + EnvironmentVariables map[string]any + TestingEnvironmentVariables map[string]any +} + +// XCTestRunMetadata contains metadata about the .xctestrun file +type xCTestRunMetadata struct { + FormatVersion int `plist:"FormatVersion"` +} + +func (data xCTestRunData) buildTestConfig(device ios.DeviceEntry, listener *TestListener) (TestConfig, error) { + testsToRun := data.TestConfig.OnlyTestIdentifiers + testsToSkip := data.TestConfig.SkipTestIdentifiers + + testEnv := make(map[string]any) + if data.TestConfig.IsUITestBundle { + maps.Copy(testEnv, data.TestConfig.EnvironmentVariables) + maps.Copy(testEnv, data.TestConfig.TestingEnvironmentVariables) + } + + // Extract only the file name + var testBundlePath = filepath.Base(data.TestConfig.TestBundlePath) + + // Build the TestConfig object from parsed data + testConfig := TestConfig{ + TestRunnerBundleId: data.TestConfig.TestHostBundleIdentifier, + XctestConfigName: testBundlePath, + Args: data.TestConfig.CommandLineArguments, + Env: testEnv, + TestsToRun: testsToRun, + TestsToSkip: testsToSkip, + XcTest: !data.TestConfig.IsUITestBundle, + Device: device, + Listener: listener, + } + + return testConfig, nil +} + +// parseFile reads the .xctestrun file and decodes it into a map +func parseFile(filePath string) (xCTestRunData, error) { + file, err := os.Open(filePath) + if err != nil { + return xCTestRunData{}, fmt.Errorf("failed to open xctestrun file: %w", err) + } + defer file.Close() + return decode(file) +} + +// decode decodes the binary xctestrun content into the xCTestRunData struct +func decode(r io.Reader) (xCTestRunData, error) { + // Read the entire content once + content, err := io.ReadAll(r) + if err != nil { + return xCTestRunData{}, fmt.Errorf("failed to read content: %w", err) + } + + // Use a single map for initial parsing + var rawData map[string]interface{} + if _, err := plist.Unmarshal(content, &rawData); err != nil { + return xCTestRunData{}, fmt.Errorf("failed to unmarshal plist: %w", err) + } + + result := xCTestRunData{ + TestConfig: schemeData{}, // Initialize TestConfig + } + + // Parse metadata + metadataMap, ok := rawData["__xctestrun_metadata__"].(map[string]interface{}) + if !ok { + return xCTestRunData{}, errors.New("invalid or missing __xctestrun_metadata__") + } + + // Direct decoding of metadata to avoid additional conversion + switch v := metadataMap["FormatVersion"].(type) { + case int: + result.XCTestRunMetadata.FormatVersion = v + case uint64: + result.XCTestRunMetadata.FormatVersion = int(v) + default: + return xCTestRunData{}, fmt.Errorf("unexpected FormatVersion type: %T", metadataMap["FormatVersion"]) + } + + // Verify FormatVersion + if result.XCTestRunMetadata.FormatVersion != 1 { + return result, fmt.Errorf("go-ios currently only supports .xctestrun files in formatVersion 1: "+ + "The formatVersion of your xctestrun file is %d, feel free to open an issue in https://github.com/danielpaulus/go-ios/issues to "+ + "add support", result.XCTestRunMetadata.FormatVersion) + } + + // Parse test schemes + if err := parseTestSchemes(rawData, &result.TestConfig); err != nil { + return xCTestRunData{}, err + } + + return result, nil +} + +// parseTestSchemes extracts and parses test schemes from the raw data +func parseTestSchemes(rawData map[string]interface{}, scheme *schemeData) error { + // Dynamically find and parse test schemes + for key, value := range rawData { + // Skip metadata key + if key == "__xctestrun_metadata__" { + continue + } + + // Attempt to convert to schemeData + schemeMap, ok := value.(map[string]interface{}) + if !ok { + continue // Skip if not a valid scheme map + } + + // Parse the scheme into schemeData and update the TestConfig + var schemeParsed schemeData + schemeBuf := new(bytes.Buffer) + encoder := plist.NewEncoder(schemeBuf) + if err := encoder.Encode(schemeMap); err != nil { + return fmt.Errorf("failed to encode scheme %s: %w", key, err) + } + + // Decode the plist buffer into schemeData + decoder := plist.NewDecoder(bytes.NewReader(schemeBuf.Bytes())) + if err := decoder.Decode(&schemeParsed); err != nil { + return fmt.Errorf("failed to decode scheme %s: %w", key, err) + } + + // Store the scheme in the result TestConfig + *scheme = schemeParsed + break // Only one scheme expected, break after the first valid scheme + } + + return nil +} diff --git a/ios/testmanagerd/xcuitestrunner.go b/ios/testmanagerd/xcuitestrunner.go index a49dbf3b..9b20269d 100644 --- a/ios/testmanagerd/xcuitestrunner.go +++ b/ios/testmanagerd/xcuitestrunner.go @@ -249,6 +249,22 @@ type TestConfig struct { Listener *TestListener } +func StartXCTestWithConfig(ctx context.Context, xctestrunFilePath string, device ios.DeviceEntry, listener *TestListener) ([]TestSuite, error) { + results, err := parseFile(xctestrunFilePath) + if err != nil { + log.Errorf("Error parsing xctestrun file: %v", err) + return nil, err + } + + testConfig, err := results.buildTestConfig(device, listener) + if err != nil { + log.Errorf("Error while constructing the test config: %v", err) + return nil, err + } + + return RunTestWithConfig(ctx, testConfig) +} + func RunTestWithConfig(ctx context.Context, testConfig TestConfig) ([]TestSuite, error) { if len(testConfig.TestRunnerBundleId) == 0 { return nil, fmt.Errorf("RunTestWithConfig: testConfig.TestRunnerBundleId can not be empty") @@ -461,11 +477,14 @@ func startTestRunner17(appserviceConn *appservice.Connection, bundleID string, s log.Debugf("adding extra env %s=%s", key, value) } } + var opts = map[string]interface{}{} - opts := map[string]interface{}{ - "ActivateSuspended": uint64(1), - "StartSuspendedKey": uint64(0), - "__ActivateSuspended": uint64(1), + if !isXCTest { + opts = map[string]interface{}{ + "ActivateSuspended": uint64(1), + "StartSuspendedKey": uint64(0), + "__ActivateSuspended": uint64(1), + } } appLaunch, err := appserviceConn.LaunchAppWithStdIo( diff --git a/main.go b/main.go index 53fb11a1..ec24ed8f 100644 --- a/main.go +++ b/main.go @@ -114,6 +114,7 @@ Usage: ios kill ( | --pid= | --process=) [options] ios memlimitoff (--process=) [options] ios runtest [--bundle-id=] [--test-runner-bundle-id=] [--xctest-config=] [--log-output=] [--xctest] [--test-to-run=]... [--test-to-skip=]... [--env=]... [options] + ios runxctest [--xctestrun-file-path=] [--log-output=] [options] ios runwda [--bundleid=] [--testrunnerbundleid=] [--xctestconfig=] [--log-output=] [--arg=]... [--env=]... [options] ios ax [--font=] [options] ios debug [options] [--stop-at-entry] @@ -232,6 +233,8 @@ The commands work as following: > If you provide '-' as log output, it prints resuts to stdout. > To be able to filter for tests to run or skip, use one argument per test selector. Example: runtest --test-to-run=(TestTarget.)TestClass/testMethod --test-to-run=(TestTarget.)TestClass/testMethod (the value for 'TestTarget' is optional) > The method name can also be omitted and in this case all tests of the specified class are run + ios runxctest [--xctestrun-file-path=] [--log-output=] [options] Run a XCTest. The --xctestrun-file-path specifies the path to the .xctestrun file to configure the test execution. + > If you provide '-' as log output, it prints resuts to stdout. ios runwda [--bundleid=] [--testrunnerbundleid=] [--xctestconfig=] [--log-output=] [--arg=]... [--env=]...[options] runs WebDriverAgents > specify runtime args and env vars like --env ENV_1=something --env ENV_2=else and --arg ARG1 --arg ARG2 ios ax [--font=] [options] Access accessibility inspector features. @@ -1001,6 +1004,38 @@ The commands work as following: return } + b, _ = arguments.Bool("runxctest") + if b { + xctestrunFilePath, _ := arguments.String("--xctestrun-file-path") + + rawTestlog, rawTestlogErr := arguments.String("--log-output") + + if rawTestlogErr == nil { + var writer *os.File = os.Stdout + if rawTestlog != "-" { + file, err := os.Create(rawTestlog) + exitIfError("Cannot open file "+rawTestlog, err) + writer = file + } + defer writer.Close() + var listener = testmanagerd.NewTestListener(writer, writer, os.TempDir()) + + testResults, err := testmanagerd.StartXCTestWithConfig(context.TODO(), xctestrunFilePath, device, listener) + if err != nil { + log.WithFields(log.Fields{"error": err}).Info("Failed running Xctest") + } + + log.Info(fmt.Printf("%+v", testResults)) + } else { + var listener = testmanagerd.NewTestListener(io.Discard, io.Discard, os.TempDir()) + _, err := testmanagerd.StartXCTestWithConfig(context.TODO(), xctestrunFilePath, device, listener) + if err != nil { + log.WithFields(log.Fields{"error": err}).Info("Failed running Xctest") + } + } + return + } + if runWdaCommand(device, arguments) { return }