Skip to content

Commit

Permalink
Swift Testing Attachments Support (#1187)
Browse files Browse the repository at this point in the history 
* Attachments Support

Support swift-testing attachments. Currently prints the attachments out as a report at the end of a test run. In order to leverage them we set the --experimental-attachment-path defined by a new swift.attachmentsPath setting. This is a directory path which can be global or scoped to the workspace. It can be relative or absolute. If relative, the absolute path is computed relative to the project workspace root. Defaults to .build/attachments.

If the attachmentsPath directory doesn't exist, it is created. Each test run that produces attachments creates a subfolder named with the date/time of the test run, and places the runs attachments within it.
  • Loading branch information
@plemarquand
plemarquand authored Jan 13, 2025
1 parent dd19e22 commit 9a4b2db
Show file tree
Hide file tree
Showing 10 changed files with 312 additions and 35 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ default.profraw
*.vsix
.vscode-test
.build
.index-build
.DS_Store
assets/documentation-webview
assets/test/**/Package.resolved
Expand Down
11 changes: 10 additions & 1 deletion assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,13 @@ final class DebugReleaseTestSuite: XCTestCase {
}
}

#if swift(>=6.0)
#if swift(>=6.1)
@_spi(Experimental) import Testing
#elseif swift(>=6.0)
import Testing
#endif

#if swift(>=6.0)
@Test func topLevelTestPassing() {
print("A print statement in a test.")
#if !TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING
Expand Down Expand Up @@ -98,7 +102,12 @@ struct MixedSwiftTestingSuite {
}
#expect(2 == 3)
}
#endif

#if swift(>=6.1)
@Test func testAttachment() throws {
Attachment("Hello, world!", named: "hello.txt").attach()
}
#endif

final class DuplicateSuffixTests: XCTestCase {
Expand Down
6 changes: 6 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,12 @@
}
}
}
},
"swift.attachmentsPath": {
"type": "string",
"default": ".build/attachments",
"markdownDescription": "The path to a directory that will be used to store attachments produced during a test run.\n\nA relative path resolves relative to the root directory of the workspace running the test(s)",
"scope": "machine-overridable"
}
}
},
Expand Down
30 changes: 27 additions & 3 deletions src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ export type EventRecordPayload =
| TestCaseEnded
| IssueRecorded
| TestSkipped
| RunEnded;
| RunEnded
| ValueAttached;

export interface EventRecord extends VersionedRecord {
kind: "event";
Expand Down Expand Up @@ -95,6 +96,15 @@ interface RunEnded {
messages: EventMessage[];
}

interface ValueAttached {
kind: "_valueAttached";
_attachment: {
path?: string;
};
testID: string;
messages: EventMessage[];
}

interface Instant {
absolute: number;
since1970: number;
Expand Down Expand Up @@ -148,6 +158,7 @@ export enum TestSymbol {
difference = "difference",
warning = "warning",
details = "details",
attachment = "attachment",
none = "none",
}

Expand All @@ -169,7 +180,8 @@ export class SwiftTestingOutputParser {

constructor(
public testRunStarted: () => void,
public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void
public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void,
public onAttachment: (testIndex: number, path: string) => void
) {}

/**
Expand Down Expand Up @@ -457,6 +469,12 @@ export class SwiftTestingOutputParser {
this.completionMap.set(testIndex, true);
runState.completed(testIndex, { timestamp: item.payload.instant.absolute });
return;
} else if (item.payload.kind === "_valueAttached" && item.payload._attachment.path) {
const testID = this.idFromOptionalTestCase(item.payload.testID);
const testIndex = this.getTestCaseIndex(runState, testID);

this.onAttachment(testIndex, item.payload._attachment.path);
return;
}
}
}
Expand Down Expand Up @@ -495,7 +513,7 @@ export class SymbolRenderer {
* @param message An event message, typically found on an `EventRecordPayload`.
* @returns A string colorized with ANSI escape codes.
*/
static eventMessageSymbol(symbol: TestSymbol): string {
public static eventMessageSymbol(symbol: TestSymbol): string {
return this.colorize(symbol, this.symbol(symbol));
}

Expand All @@ -521,6 +539,8 @@ export class SymbolRenderer {
return "\u{25B2}"; // Unicode: BLACK UP-POINTING TRIANGLE
case TestSymbol.details:
return "\u{2192}"; // Unicode: RIGHTWARDS ARROW
case TestSymbol.attachment:
return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
case TestSymbol.none:
return "";
}
Expand All @@ -540,6 +560,8 @@ export class SymbolRenderer {
return "\u{26A0}\u{FE0E}"; // Unicode: WARNING SIGN + VARIATION SELECTOR-15 (disable emoji)
case TestSymbol.details:
return "\u{21B3}"; // Unicode: DOWNWARDS ARROW WITH TIP RIGHTWARDS
case TestSymbol.attachment:
return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
case TestSymbol.none:
return " ";
}
Expand All @@ -562,6 +584,8 @@ export class SymbolRenderer {
return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.warning:
return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.attachment:
return `${SymbolRenderer.ansiEscapeCodePrefix}94m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
case TestSymbol.none:
default:
return symbol;
Expand Down
108 changes: 90 additions & 18 deletions src/TestExplorer/TestRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ import {
ParallelXCTestOutputParser,
XCTestOutputParser,
} from "./TestParsers/XCTestOutputParser";
import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
import {
SwiftTestingOutputParser,
SymbolRenderer,
TestSymbol,
} from "./TestParsers/SwiftTestingOutputParser";
import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
import { TaskOperation } from "../tasks/TaskQueue";
import { TestXUnitParser } from "./TestXUnitParser";
Expand All @@ -36,7 +40,12 @@ import { TestRunArguments } from "./TestRunArguments";
import { TemporaryFolder } from "../utilities/tempFolder";
import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
import { TestCoverage } from "../coverage/LcovResults";
import { BuildConfigurationFactory, TestingConfigurationFactory } from "../debugger/buildConfig";
import {
BuildConfigurationFactory,
SwiftTestingBuildAguments,
SwiftTestingConfigurationSetup,
TestingConfigurationFactory,
} from "../debugger/buildConfig";
import { TestKind, isDebugging, isRelease } from "./TestKind";
import { reduceTestItemChildren } from "./TestUtils";
import { CompositeCancellationToken } from "../utilities/cancellation";
Expand Down Expand Up @@ -67,6 +76,7 @@ export class TestRunProxy {
private queuedOutput: string[] = [];
private _testItems: vscode.TestItem[];
private iteration: number | undefined;
private attachments: { [key: string]: string[] } = {};
public coverage: TestCoverage;
public token: CompositeCancellationToken;

Expand Down Expand Up @@ -177,6 +187,12 @@ export class TestRunProxy {
this.addedTestItems.push({ testClass, parentIndex });
};

public addAttachment = (testIndex: number, attachment: string) => {
const attachments = this.attachments[testIndex] ?? [];
attachments.push(attachment);
this.attachments[testIndex] = attachments;
};

public getTestIndex(id: string, filename?: string): number {
return this.testItemFinder.getIndex(id, filename);
}
Expand Down Expand Up @@ -231,6 +247,8 @@ export class TestRunProxy {
if (!this.runStarted) {
this.testRunStarted();
}

this.reportAttachments();
this.testRun?.end();
this.testRunCompleteEmitter.fire();
this.token.dispose();
Expand Down Expand Up @@ -259,6 +277,25 @@ export class TestRunProxy {
}
}

private reportAttachments() {
const attachmentKeys = Object.keys(this.attachments);
if (attachmentKeys.length > 0) {
let attachment = "";
const totalAttachments = attachmentKeys.reduce((acc, key) => {
const attachments = this.attachments[key];
attachment = attachments.length ? attachments[0] : attachment;
return acc + attachments.length;
}, 0);

if (attachment) {
attachment = path.dirname(attachment);
this.appendOutput(
`${SymbolRenderer.eventMessageSymbol(TestSymbol.attachment)} ${SymbolRenderer.ansiEscapeCodePrefix}90mRecorded ${totalAttachments} attachment${totalAttachments === 1 ? "" : "s"} to ${attachment}${SymbolRenderer.resetANSIEscapeCode}`
);
}
}
}

private performAppendOutput(
testRun: vscode.TestRun,
output: string,
Expand Down Expand Up @@ -321,7 +358,8 @@ export class TestRunner {
: new XCTestOutputParser();
this.swiftTestOutputParser = new SwiftTestingOutputParser(
this.testRun.testRunStarted,
this.testRun.addParameterizedTestCase
this.testRun.addParameterizedTestCase,
this.testRun.addAttachment
);
}

Expand All @@ -334,7 +372,8 @@ export class TestRunner {
// The SwiftTestingOutputParser holds state and needs to be reset between iterations.
this.swiftTestOutputParser = new SwiftTestingOutputParser(
this.testRun.testRunStarted,
this.testRun.addParameterizedTestCase
this.testRun.addParameterizedTestCase,
this.testRun.addAttachment
);
this.testRun.setIteration(iteration);
}
Expand Down Expand Up @@ -519,18 +558,28 @@ export class TestRunner {
// Run swift-testing first, then XCTest.
// swift-testing being parallel by default should help these run faster.
if (this.testArgs.hasSwiftTestingTests) {
const fifoPipePath = this.generateFifoPipePath();
const testRunTime = Date.now();
const fifoPipePath = this.generateFifoPipePath(testRunTime);

await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
// macOS/Linux require us to create the named pipe before we use it.
// Windows just lets us communicate by specifying a pipe path without any ceremony.
if (process.platform !== "win32") {
await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
}

const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
// Create the swift-testing configuration JSON file, peparing any
// directories the configuration may require.
const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
this.folderContext,
testRunTime
);
const swiftTestingArgs = await SwiftTestingBuildAguments.build(
fifoPipePath,
attachmentFolder
);
const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
this.folderContext,
swiftTestingArgs,
this.testKind,
this.testArgs.swiftTestArgs,
true
Expand All @@ -540,17 +589,25 @@ export class TestRunner {
return this.testRun.runState;
}

const outputStream = this.testOutputWritable(TestLibrary.swiftTesting, runState);

// Watch the pipe for JSONL output and parse the events into test explorer updates.
// The await simply waits for the watching to be configured.
await this.swiftTestOutputParser.watch(fifoPipePath, runState);

await this.launchTests(
runState,
this.testKind === TestKind.parallel ? TestKind.standard : this.testKind,
this.testOutputWritable(TestLibrary.swiftTesting, runState),
outputStream,
testBuildConfig,
TestLibrary.swiftTesting
);

await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
this.folderContext,
testRunTime,
this.workspaceContext.outputChannel
);
});
}

Expand Down Expand Up @@ -774,21 +831,32 @@ export class TestRunner {
throw new Error(`Build failed with exit code ${buildExitCode}`);
}

const testRunTime = Date.now();
const subscriptions: vscode.Disposable[] = [];
const buildConfigs: Array<vscode.DebugConfiguration | undefined> = [];
const fifoPipePath = this.generateFifoPipePath();
const fifoPipePath = this.generateFifoPipePath(testRunTime);

await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
if (this.testArgs.hasSwiftTestingTests) {
// macOS/Linux require us to create the named pipe before we use it.
// Windows just lets us communicate by specifying a pipe path without any ceremony.
if (process.platform !== "win32") {
await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
}
// Create the swift-testing configuration JSON file, peparing any
// directories the configuration may require.
const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
this.folderContext,
testRunTime
);
const swiftTestingArgs = await SwiftTestingBuildAguments.build(
fifoPipePath,
attachmentFolder
);

const swiftTestBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
this.folderContext,
fifoPipePath,
swiftTestingArgs,
this.testKind,
this.testArgs.swiftTestArgs,
true
Expand Down Expand Up @@ -930,9 +998,6 @@ export class TestRunner {
}
},
reason => {
this.workspaceContext.outputChannel.logDiagnostic(
`Failed to debug test: ${reason}`
);
subscriptions.forEach(sub => sub.dispose());
reject(reason);
}
Expand All @@ -942,6 +1007,13 @@ export class TestRunner {

// Run each debugging session sequentially
await debugRuns.reduce((p, fn) => p.then(() => fn()), Promise.resolve());

// Clean up any leftover resources
await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
this.folderContext,
testRunTime,
this.workspaceContext.outputChannel
);
});
}

Expand Down Expand Up @@ -996,10 +1068,10 @@ export class TestRunner {
}
}

private generateFifoPipePath(): string {
private generateFifoPipePath(testRunDateNow: number): string {
return process.platform === "win32"
? `\\\\.\\pipe\\vscodemkfifo-${Date.now()}`
: path.join(os.tmpdir(), `vscodemkfifo-${Date.now()}`);
? `\\\\.\\pipe\\vscodemkfifo-${testRunDateNow}`
: path.join(os.tmpdir(), `vscodemkfifo-${testRunDateNow}`);
}
}

Expand Down
7 changes: 7 additions & 0 deletions src/configuration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ export interface FolderConfiguration {
readonly autoGenerateLaunchConfigurations: boolean;
/** disable automatic running of swift package resolve */
readonly disableAutoResolve: boolean;
/** location to save swift-testing attachments */
readonly attachmentsPath: string;
/** look up saved permissions for the supplied plugin */
pluginPermissions(pluginId: string): PluginPermissionConfiguration;
}
Expand Down Expand Up @@ -162,6 +164,11 @@ const configuration = {
.getConfiguration("swift", workspaceFolder)
.get<boolean>("searchSubfoldersForPackages", false);
},
get attachmentsPath(): string {
return vscode.workspace
.getConfiguration("swift", workspaceFolder)
.get<string>("attachmentsPath", "./.build/attachments");
},
pluginPermissions(pluginId: string): PluginPermissionConfiguration {
return (
vscode.workspace.getConfiguration("swift", workspaceFolder).get<{
Expand Down
Loading

0 comments on commit 9a4b2db

Please sign in to comment.