diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs new file mode 100644 index 000000000000..23610937bb29 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs @@ -0,0 +1,35 @@ +import * as Sentry from '@sentry/node'; +import { embed, embedMany } from 'ai'; +import { MockEmbeddingModelV1 } from 'ai/test'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Single embedding + await embed({ + model: new MockEmbeddingModelV1({ + doEmbed: async () => ({ + embeddings: [[0.1, 0.2, 0.3]], + usage: { tokens: 10 }, + }), + }), + value: 'Embedding test!', + }); + + // Multiple embeddings + await embedMany({ + model: new MockEmbeddingModelV1({ + maxEmbeddingsPerCall: 5, + doEmbed: async () => ({ + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + usage: { tokens: 20 }, + }), + }), + values: ['First input', 'Second input'], + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 2919815b8f0d..03984389b797 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -2,6 +2,7 @@ import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from ' import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -830,4 +831,90 @@ describe('Vercel AI integration', () => { }); }, ); + + createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates embedding related spans with sendDefaultPii: false', async () => { + const expectedTransaction = { + transaction: 'main', + spans: expect.arrayContaining([ + // embed doEmbed span + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, + }), + description: 'embeddings mock-model-id', + op: 'gen_ai.embeddings', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // embedMany doEmbed span + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, + }), + description: 'embeddings mock-model-id', + op: 'gen_ai.embeddings', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + + await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates embedding related spans with sendDefaultPii: true', async () => { + const expectedTransaction = { + transaction: 'main', + spans: expect.arrayContaining([ + // embed doEmbed span with input + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', + [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Embedding test!', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, + }), + description: 'embeddings mock-model-id', + op: 'gen_ai.embeddings', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // embedMany doEmbed span with input + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', + [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: '["First input","Second input"]', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, + }), + description: 'embeddings mock-model-id', + op: 'gen_ai.embeddings', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + + await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + }); + }); }); diff --git a/packages/core/src/tracing/ai/gen-ai-attributes.ts b/packages/core/src/tracing/ai/gen-ai-attributes.ts index dc88e6315852..8840843e0ae1 100644 --- a/packages/core/src/tracing/ai/gen-ai-attributes.ts +++ b/packages/core/src/tracing/ai/gen-ai-attributes.ts @@ -227,12 +227,12 @@ export const GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE = 'gen_ai.embeddings.input'; /** * The span operation name for embedding */ -export const GEN_AI_EMBED_DO_EMBED_OPERATION_ATTRIBUTE = 'gen_ai.embed'; +export const GEN_AI_EMBED_DO_EMBED_OPERATION_ATTRIBUTE = 'gen_ai.embeddings'; /** * The span operation name for embedding many */ -export const GEN_AI_EMBED_MANY_DO_EMBED_OPERATION_ATTRIBUTE = 'gen_ai.embed_many'; +export const GEN_AI_EMBED_MANY_DO_EMBED_OPERATION_ATTRIBUTE = 'gen_ai.embeddings'; /** * The span operation name for reranking diff --git a/packages/core/src/tracing/vercel-ai/constants.ts b/packages/core/src/tracing/vercel-ai/constants.ts index 94561dae3e98..fb82c6063dd4 100644 --- a/packages/core/src/tracing/vercel-ai/constants.ts +++ b/packages/core/src/tracing/vercel-ai/constants.ts @@ -6,15 +6,7 @@ import type { ToolCallSpanContext } from './types'; export const toolCallSpanContextMap = new Map(); // Operation sets for efficient mapping to OpenTelemetry semantic convention values -export const INVOKE_AGENT_OPS = new Set([ - 'ai.generateText', - 'ai.streamText', - 'ai.generateObject', - 'ai.streamObject', - 'ai.embed', - 'ai.embedMany', - 'ai.rerank', -]); +export const INVOKE_AGENT_OPS = new Set(['ai.generateText', 'ai.streamText', 'ai.generateObject', 'ai.streamObject']); export const GENERATE_CONTENT_OPS = new Set([ 'ai.generateText.doGenerate', @@ -28,7 +20,7 @@ export const EMBEDDINGS_OPS = new Set(['ai.embed.doEmbed', 'ai.embedMany.doEmbed export const RERANK_OPS = new Set(['ai.rerank.doRerank']); export const DO_SPAN_NAME_PREFIX: Record = { - 'ai.embed.doEmbed': 'embed', - 'ai.embedMany.doEmbed': 'embed_many', + 'ai.embed.doEmbed': 'embeddings', + 'ai.embedMany.doEmbed': 'embeddings', 'ai.rerank.doRerank': 'rerank', }; diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index f6fbee6d68f5..43f019dc78a1 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -5,6 +5,7 @@ import type { Event } from '../../types-hoist/event'; import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON } from '../../types-hoist/span'; import { spanToJSON } from '../../utils/spanUtils'; import { + GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, @@ -55,6 +56,8 @@ import { AI_USAGE_CACHED_INPUT_TOKENS_ATTRIBUTE, AI_USAGE_COMPLETION_TOKENS_ATTRIBUTE, AI_USAGE_PROMPT_TOKENS_ATTRIBUTE, + AI_USAGE_TOKENS_ATTRIBUTE, + AI_VALUES_ATTRIBUTE, OPERATION_NAME_ATTRIBUTE, } from './vercel-ai-attributes'; @@ -160,6 +163,9 @@ function processEndedVercelAiSpan(span: SpanJSON): void { renameAttributeKey(attributes, 'ai.usage.inputTokens', GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE); renameAttributeKey(attributes, 'ai.usage.outputTokens', GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE); + // Embedding spans use ai.usage.tokens instead of promptTokens/completionTokens + renameAttributeKey(attributes, AI_USAGE_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE); + // AI SDK uses avgOutputTokensPerSecond, map to our expected attribute name renameAttributeKey(attributes, 'ai.response.avgOutputTokensPerSecond', 'ai.response.avgCompletionTokensPerSecond'); @@ -172,12 +178,13 @@ function processEndedVercelAiSpan(span: SpanJSON): void { attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE] + attributes[GEN_AI_USAGE_INPUT_TOKENS_CACHED_ATTRIBUTE]; } - if ( - typeof attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] === 'number' && - typeof attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE] === 'number' - ) { - attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE] = - attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] + attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]; + // Compute total tokens from input + output (embeddings may only have input tokens) + if (typeof attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE] === 'number') { + const outputTokens = + typeof attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] === 'number' + ? attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE] + : 0; + attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE] = outputTokens + attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]; } // Convert the available tools array to a JSON string @@ -207,6 +214,20 @@ function processEndedVercelAiSpan(span: SpanJSON): void { renameAttributeKey(attributes, AI_SCHEMA_ATTRIBUTE, 'gen_ai.request.schema'); renameAttributeKey(attributes, AI_MODEL_ID_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE); + // Map embedding input: ai.values → gen_ai.embeddings.input + // Vercel AI SDK JSON-stringifies each value individually, so we parse each element back. + // Single embed gets unwrapped to a plain value; batch embedMany stays as a JSON array. + if (Array.isArray(attributes[AI_VALUES_ATTRIBUTE])) { + const parsed = (attributes[AI_VALUES_ATTRIBUTE] as string[]).map(v => { + try { + return JSON.parse(v); + } catch { + return v; + } + }); + attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE] = parsed.length === 1 ? parsed[0] : JSON.stringify(parsed); + } + addProviderMetadataToAttributes(attributes); // Change attributes namespaced with `ai.X` to `vercel.ai.X` diff --git a/packages/core/src/tracing/vercel-ai/utils.ts b/packages/core/src/tracing/vercel-ai/utils.ts index 139d75a241ee..94f5dd5f7b10 100644 --- a/packages/core/src/tracing/vercel-ai/utils.ts +++ b/packages/core/src/tracing/vercel-ai/utils.ts @@ -235,9 +235,6 @@ export function getSpanOpFromName(name: string): string | undefined { case 'ai.streamText': case 'ai.generateObject': case 'ai.streamObject': - case 'ai.embed': - case 'ai.embedMany': - case 'ai.rerank': return GEN_AI_INVOKE_AGENT_OPERATION_ATTRIBUTE; case 'ai.generateText.doGenerate': return GEN_AI_GENERATE_TEXT_DO_GENERATE_OPERATION_ATTRIBUTE; diff --git a/packages/core/test/lib/tracing/vercel-ai-rerank.test.ts b/packages/core/test/lib/tracing/vercel-ai-rerank.test.ts index 7deb331020c3..8bc30b89c264 100644 --- a/packages/core/test/lib/tracing/vercel-ai-rerank.test.ts +++ b/packages/core/test/lib/tracing/vercel-ai-rerank.test.ts @@ -3,8 +3,8 @@ import { getSpanOpFromName } from '../../../src/tracing/vercel-ai/utils'; describe('vercel-ai rerank support', () => { describe('getSpanOpFromName', () => { - it('should map ai.rerank to gen_ai.invoke_agent', () => { - expect(getSpanOpFromName('ai.rerank')).toBe('gen_ai.invoke_agent'); + it('should not assign a gen_ai op to ai.rerank pipeline span', () => { + expect(getSpanOpFromName('ai.rerank')).toBeUndefined(); }); it('should map ai.rerank.doRerank to gen_ai.rerank', () => {