Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add telemetry events for free AI credits feature (no-changelog) #12459

Merged
merged 3 commits into from
Jan 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions packages/cli/src/events/__tests__/telemetry-event-relay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import type { INode, INodesGraphResult } from 'n8n-workflow';
import { NodeApiError, TelemetryHelpers, type IRun, type IWorkflowBase } from 'n8n-workflow';

import { N8N_VERSION } from '@/constants';
import type { CredentialsEntity } from '@/databases/entities/credentials-entity';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
import type { CredentialsRepository } from '@/databases/repositories/credentials.repository';
import type { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository';
import type { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository';
import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
Expand Down Expand Up @@ -52,6 +54,7 @@ describe('TelemetryEventRelay', () => {
const nodeTypes = mock<NodeTypes>();
const sharedWorkflowRepository = mock<SharedWorkflowRepository>();
const projectRelationRepository = mock<ProjectRelationRepository>();
const credentialsRepository = mock<CredentialsRepository>();
const eventService = new EventService();

let telemetryEventRelay: TelemetryEventRelay;
Expand All @@ -67,6 +70,7 @@ describe('TelemetryEventRelay', () => {
nodeTypes,
sharedWorkflowRepository,
projectRelationRepository,
credentialsRepository,
);

await telemetryEventRelay.init();
Expand All @@ -90,6 +94,7 @@ describe('TelemetryEventRelay', () => {
nodeTypes,
sharedWorkflowRepository,
projectRelationRepository,
credentialsRepository,
);
// @ts-expect-error Private method
const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners');
Expand All @@ -112,6 +117,7 @@ describe('TelemetryEventRelay', () => {
nodeTypes,
sharedWorkflowRepository,
projectRelationRepository,
credentialsRepository,
);
// @ts-expect-error Private method
const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners');
Expand Down Expand Up @@ -1197,6 +1203,9 @@ describe('TelemetryEventRelay', () => {

it('should call telemetry.track when manual node execution finished', async () => {
sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor');
credentialsRepository.findOneBy.mockResolvedValue(
mock<CredentialsEntity>({ type: 'openAiApi', isManaged: false }),
);

const runData = {
status: 'error',
Expand Down Expand Up @@ -1276,6 +1285,8 @@ describe('TelemetryEventRelay', () => {
error_node_id: '1',
node_id: '1',
node_type: 'n8n-nodes-base.jira',
is_managed: false,
credential_type: null,
node_graph_string: JSON.stringify(nodeGraph.nodeGraph),
}),
);
Expand Down Expand Up @@ -1498,5 +1509,118 @@ describe('TelemetryEventRelay', () => {
}),
);
});

it('should call telemetry.track when manual node execution finished with is_managed and credential_type properties', async () => {
sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor');
credentialsRepository.findOneBy.mockResolvedValue(
mock<CredentialsEntity>({ type: 'openAiApi', isManaged: true }),
);

const runData = {
status: 'error',
mode: 'manual',
data: {
executionData: {
nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }],
},
startData: {
destinationNode: 'OpenAI',
runNodeFilter: ['OpenAI'],
},
resultData: {
runData: {},
lastNodeExecuted: 'OpenAI',
error: new NodeApiError(
{
id: '1',
typeVersion: 1,
name: 'Jira',
type: 'n8n-nodes-base.jira',
parameters: {},
position: [100, 200],
},
{
message: 'Error message',
description: 'Incorrect API key provided',
httpCode: '401',
stack: '',
},
{
message: 'Error message',
description: 'Error description',
level: 'warning',
functionality: 'regular',
},
),
},
},
} as unknown as IRun;

const nodeGraph: INodesGraphResult = {
nodeGraph: { node_types: [], node_connections: [], webhookNodeNames: [] },
nameIndices: {
Jira: '1',
OpenAI: '1',
},
} as unknown as INodesGraphResult;

jest.spyOn(TelemetryHelpers, 'generateNodesGraph').mockImplementation(() => nodeGraph);

jest
.spyOn(TelemetryHelpers, 'getNodeTypeForName')
.mockImplementation(
() => ({ type: 'n8n-nodes-base.jira', version: 1, name: 'Jira' }) as unknown as INode,
);

const event: RelayEventMap['workflow-post-execute'] = {
workflow: mockWorkflowBase,
executionId: 'execution123',
userId: 'user123',
runData,
};

eventService.emit('workflow-post-execute', event);

await flushPromises();

expect(credentialsRepository.findOneBy).toHaveBeenCalledWith({
id: 'nhu-l8E4hX',
});

expect(telemetry.track).toHaveBeenCalledWith(
'Manual node exec finished',
expect.objectContaining({
webhook_domain: null,
user_id: 'user123',
workflow_id: 'workflow123',
status: 'error',
executionStatus: 'error',
sharing_role: 'sharee',
error_message: 'Error message',
error_node_type: 'n8n-nodes-base.jira',
error_node_id: '1',
node_id: '1',
node_type: 'n8n-nodes-base.jira',

is_managed: true,
credential_type: 'openAiApi',
node_graph_string: JSON.stringify(nodeGraph.nodeGraph),
}),
);

expect(telemetry.trackWorkflowExecution).toHaveBeenCalledWith(
expect.objectContaining({
workflow_id: 'workflow123',
success: false,
is_manual: true,
execution_mode: 'manual',
version_cli: N8N_VERSION,
error_message: 'Error message',
error_node_type: 'n8n-nodes-base.jira',
node_graph_string: JSON.stringify(nodeGraph.nodeGraph),
error_node_id: '1',
}),
);
});
});
});
17 changes: 16 additions & 1 deletion packages/cli/src/events/relays/telemetry.event-relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { Service } from 'typedi';

import config from '@/config';
import { N8N_VERSION } from '@/constants';
import { CredentialsRepository } from '@/databases/repositories/credentials.repository';
import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository';
import { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository';
import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
Expand All @@ -34,6 +35,7 @@ export class TelemetryEventRelay extends EventRelay {
private readonly nodeTypes: NodeTypes,
private readonly sharedWorkflowRepository: SharedWorkflowRepository,
private readonly projectRelationRepository: ProjectRelationRepository,
private readonly credentialsRepository: CredentialsRepository,
) {
super(eventService);
}
Expand Down Expand Up @@ -693,6 +695,8 @@ export class TelemetryEventRelay extends EventRelay {
error_node_id: telemetryProperties.error_node_id as string,
webhook_domain: null,
sharing_role: userRole,
credential_type: null,
is_managed: false,
};

if (!manualExecEventProperties.node_graph_string) {
Expand All @@ -703,7 +707,18 @@ export class TelemetryEventRelay extends EventRelay {
}

if (runData.data.startData?.destinationNode) {
const telemetryPayload = {
const credentialsData = TelemetryHelpers.extractLastExecutedNodeCredentialData(runData);
if (credentialsData) {
manualExecEventProperties.credential_type = credentialsData.credentialType;
const credential = await this.credentialsRepository.findOneBy({
id: credentialsData.credentialId,
});
if (credential) {
manualExecEventProperties.is_managed = credential.isManaged;
}
}

const telemetryPayload: ITelemetryTrackProperties = {
...manualExecEventProperties,
node_type: TelemetryHelpers.getNodeTypeForName(
workflow,
Expand Down
10 changes: 10 additions & 0 deletions packages/editor-ui/src/components/FreeAiCreditsCallout.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@ import { useRootStore } from '@/stores/root.store';
import { useToast } from '@/composables/useToast';
import { renderComponent } from '@/__tests__/render';
import { mockedStore } from '@/__tests__/utils';
import { useTelemetry } from '@/composables/useTelemetry';

vi.mock('@/composables/useToast', () => ({
useToast: vi.fn(),
}));

vi.mock('@/composables/useTelemetry', () => ({
useTelemetry: vi.fn(),
}));

vi.mock('@/stores/settings.store', () => ({
useSettingsStore: vi.fn(),
}));
Expand Down Expand Up @@ -100,6 +105,10 @@ describe('FreeAiCreditsCallout', () => {
(useToast as any).mockReturnValue({
showError: vi.fn(),
});

(useTelemetry as any).mockReturnValue({
track: vi.fn(),
});
});

it('should shows the claim callout when the user can claim credits', () => {
Expand All @@ -120,6 +129,7 @@ describe('FreeAiCreditsCallout', () => {
await fireEvent.click(claimButton);

expect(credentialsStore.claimFreeAiCredits).toHaveBeenCalledWith('test-project-id');
expect(useTelemetry().track).toHaveBeenCalledWith('User claimed OpenAI credits');
assertUserClaimedCredits();
});

Expand Down
4 changes: 4 additions & 0 deletions packages/editor-ui/src/components/FreeAiCreditsCallout.vue
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<script lang="ts" setup>
import { useI18n } from '@/composables/useI18n';
import { useTelemetry } from '@/composables/useTelemetry';
import { useToast } from '@/composables/useToast';
import { AI_CREDITS_EXPERIMENT } from '@/constants';
import { useCredentialsStore } from '@/stores/credentials.store';
Expand Down Expand Up @@ -32,6 +33,7 @@ const credentialsStore = useCredentialsStore();
const usersStore = useUsersStore();
const ndvStore = useNDVStore();
const projectsStore = useProjectsStore();
const telemetry = useTelemetry();

const i18n = useI18n();
const toast = useToast();
Expand Down Expand Up @@ -73,6 +75,8 @@ const onClaimCreditsClicked = async () => {
usersStore.currentUser.settings.userClaimedAiCredits = true;
}

telemetry.track('User claimed OpenAI credits');

showSuccessCallout.value = true;
} catch (e) {
toast.showError(
Expand Down
19 changes: 19 additions & 0 deletions packages/workflow/src/TelemetryHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import type {
IDataObject,
IRunData,
ITaskData,
IRun,
} from './Interfaces';
import { getNodeParameters } from './NodeHelpers';

Expand Down Expand Up @@ -470,3 +471,21 @@ export function generateNodesGraph(

return { nodeGraph, nameIndices, webhookNodeNames };
}

export function extractLastExecutedNodeCredentialData(
runData: IRun,
): null | { credentialId: string; credentialType: string } {
const nodeCredentials = runData?.data?.executionData?.nodeExecutionStack?.[0]?.node?.credentials;

if (!nodeCredentials) return null;

const credentialType = Object.keys(nodeCredentials)[0] ?? null;

if (!credentialType) return null;

const { id } = nodeCredentials[credentialType];

if (!id) return null;

return { credentialId: id, credentialType };
}
47 changes: 46 additions & 1 deletion packages/workflow/test/TelemetryHelpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import { v5 as uuidv5, v3 as uuidv3, v4 as uuidv4, v1 as uuidv1 } from 'uuid';

import { STICKY_NODE_TYPE } from '@/Constants';
import { ApplicationError } from '@/errors';
import type { IRunData } from '@/Interfaces';
import type { IRun, IRunData } from '@/Interfaces';
import { NodeConnectionType, type IWorkflowBase } from '@/Interfaces';
import * as nodeHelpers from '@/NodeHelpers';
import {
ANONYMIZATION_CHARACTER as CHAR,
extractLastExecutedNodeCredentialData,
generateNodesGraph,
getDomainBase,
getDomainPath,
Expand Down Expand Up @@ -885,6 +886,50 @@ describe('generateNodesGraph', () => {
});
});

describe('extractLastExecutedNodeCredentialData', () => {
const cases: Array<[string, IRun]> = [
['no data', mock<IRun>({ data: {} })],
['no executionData', mock<IRun>({ data: { executionData: undefined } })],
[
'no nodeExecutionStack',
mock<IRun>({ data: { executionData: { nodeExecutionStack: undefined } } }),
],
[
'no node',
mock<IRun>({
data: { executionData: { nodeExecutionStack: [{ node: undefined }] } },
}),
],
[
'no credentials',
mock<IRun>({
data: { executionData: { nodeExecutionStack: [{ node: { credentials: undefined } }] } },
}),
],
];

test.each(cases)(
'should return credentialId and credentialsType with null if %s',
(_, runData) => {
expect(extractLastExecutedNodeCredentialData(runData)).toBeNull();
},
);

it('should return correct credentialId and credentialsType when last node executed has credential', () => {
const runData = mock<IRun>({
data: {
executionData: {
nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }],
},
},
});

expect(extractLastExecutedNodeCredentialData(runData)).toMatchObject(
expect.objectContaining({ credentialId: 'nhu-l8E4hX', credentialType: 'openAiApi' }),
);
});
});

function validUrls(idMaker: typeof alphanumericId | typeof email, char = CHAR) {
const firstId = idMaker();
const secondId = idMaker();
Expand Down
Loading