Improved dev tracing P1

packages/constants/src/tools.ts

@@ -7,7 +7,7 @@ import {
   MessageRole,
   ToolMessage,
 } from './legacyCompiler'
-import { ToolCallContent as ToolRequest } from 'promptl-ai'
+import type { ToolCallContent as ToolRequest } from 'promptl-ai'
 import { StreamType, ToolCallResponse, VercelProviderTool } from './index'
 import { ToolSource, ToolSourceData } from './toolSources'
 import type { Tool } from 'ai'

packages/constants/src/tracing/index.ts

@@ -41,6 +41,8 @@ export const ATTR_LATITUDE_INTERNAL = `${ATTR_LATITUDE}.internal`
 
 export const ATTR_LATITUDE_TYPE = `${ATTR_LATITUDE}.type`
 
+export const ATTR_LATITUDE_PROMPT_PATH = `${ATTR_LATITUDE}.promptPath`
+
 export const GEN_AI_TOOL_TYPE_VALUE_FUNCTION = 'function'
 export const ATTR_GEN_AI_TOOL_CALL_ARGUMENTS = 'gen_ai.tool.call.arguments'
 export const ATTR_GEN_AI_TOOL_RESULT_VALUE = 'gen_ai.tool.result.value'
@@ -139,9 +141,13 @@ export const ATTR_LLM_RESPONSE_STOP_REASON = 'llm.response.stop_reason'
 export const ATTR_AI_OPERATION_ID = 'ai.operationId'
 export const AI_OPERATION_ID_VALUE_TOOL = 'ai.toolCall'
 export const AI_OPERATION_ID_VALUE_GENERATE_TEXT = 'ai.generateText'
+export const AI_OPERATION_ID_VALUE_GENERATE_TEXT_DO_GENERATE = 'ai.generateText.doGenerate' // prettier-ignore
 export const AI_OPERATION_ID_VALUE_STREAM_TEXT = 'ai.streamText'
+export const AI_OPERATION_ID_VALUE_STREAM_TEXT_DO_STREAM = 'ai.streamText.doStream' // prettier-ignore
 export const AI_OPERATION_ID_VALUE_GENERATE_OBJECT = 'ai.generateObject'
+export const AI_OPERATION_ID_VALUE_GENERATE_OBJECT_DO_GENERATE = 'ai.generateObject.doGenerate' // prettier-ignore
 export const AI_OPERATION_ID_VALUE_STREAM_OBJECT = 'ai.streamObject'
+export const AI_OPERATION_ID_VALUE_STREAM_OBJECT_DO_STREAM = 'ai.streamObject.doStream' // prettier-ignore
 
 export const ATTR_AI_TOOL_CALL_NAME = 'ai.toolCall.name'
 export const ATTR_AI_TOOL_CALL_ID = 'ai.toolCall.id'

packages/constants/src/tracing/span.ts

@@ -1,5 +1,5 @@
 import { FinishReason } from 'ai'
-import { Message } from 'promptl-ai'
+import type { Message } from 'promptl-ai'
 import { LogSources } from '../models'
 
 export enum SpanKind {
@@ -134,14 +134,15 @@ export type ToolSpanMetadata = BaseSpanMetadata<SpanType.Tool> & {
 }
 
 export type PromptSpanMetadata = BaseSpanMetadata<SpanType.Prompt> & {
-  experimentUuid: string
-  externalId: string
+  experimentUuid?: string
+  externalId?: string
   parameters: Record<string, unknown>
-  promptUuid: string
-  template: string
+  promptUuid?: string // Document UUID (may be resolved from promptPath server-side)
+  promptPath?: string // Path-based prompt identification
+  template?: string
   versionUuid: string
-  source: LogSources
-  projectId: number
+  source?: LogSources
+  projectId?: number
 }
 
 export type CompletionSpanMetadata = BaseSpanMetadata<SpanType.Completion> & {

packages/core/src/events/handlers/evaluateLiveLog.ts

@@ -50,6 +50,11 @@ export const evaluateLiveLogJob = async ({
     return
   }
 
+  // Cannot evaluate logs without a document UUID
+  if (!promptSpanMetadata.promptUuid) {
+    return
+  }
+
   const commitsRepository = new CommitsRepository(workspace.id)
   const commit = await commitsRepository
     .getCommitByUuid({ uuid: promptSpanMetadata.versionUuid })

packages/core/src/events/handlers/requestDocumentSuggestionJob.ts

@@ -29,7 +29,8 @@ export const requestDocumentSuggestionJobV2 = async ({
   if (!workspace) throw new NotFoundError(`Workspace not found ${workspaceId}`)
   if (result.hasPassed || result.error || result.usedForSuggestion) return
   if (!evaluation.enableSuggestions) return
-  if (!LIVE_SUGGESTION_SOURCES.includes((metadata as PromptSpanMetadata).source)) return // prettier-ignore
+  const promptMetadata = metadata as PromptSpanMetadata
+  if (!promptMetadata.source || !LIVE_SUGGESTION_SOURCES.includes(promptMetadata.source)) return // prettier-ignore
 
   const { documentSuggestionsQueue } = await queues()
   documentSuggestionsQueue.add(

packages/core/src/services/tracing/spans/completion.ts

@@ -237,6 +237,22 @@ function extractConfiguration(
   return Result.ok(toCamelCase(configuration))
 }
 
+/**
+ * Safely parse JSON payloads which can come either as JSON string or an object.
+ */
+function parseJsonPayload(value: unknown): Record<string, unknown> {
+  if (value === undefined || value === null) return {}
+  if (typeof value === 'object') return value as Record<string, unknown>
+  if (typeof value === 'string') {
+    try {
+      return JSON.parse(value)
+    } catch {
+      return {}
+    }
+  }
+  return {}
+}
+
 function convertToolCalls(
   raws: Record<string, unknown>[],
 ): TypedResult<ToolCallContent[]> {
@@ -252,14 +268,18 @@ function convertToolCalls(
           type: ContentType.toolCall,
           toolCallId: String(toolCall.id || ''),
           toolName: String(func.name || ''),
-          toolArguments: JSON.parse(String(func.arguments || '{}')),
+          toolArguments: parseJsonPayload(func.arguments),
         })
       } else {
+        // Handles multiple formats:
+        // - OpenAI: { id, name, arguments (string) }
+        // - Anthropic: { toolUseId, toolName, input (object) }
+        // - Vercel AI SDK: { toolCallId, toolName, input (object) }
         toolCalls.push({
           type: ContentType.toolCall,
           toolCallId: String(toolCall.id || toolCall.toolCallId || toolCall.toolUseId || ''), // prettier-ignore
           toolName: String(toolCall.name || toolCall.toolName || ''),
-          toolArguments: JSON.parse(String(toolCall.arguments || toolCall.toolArguments || toolCall.input || '{}')), // prettier-ignore
+          toolArguments: parseJsonPayload(toolCall.arguments || toolCall.toolArguments || toolCall.input), // prettier-ignore
         })
       }
     }

packages/core/src/services/tracing/spans/process.ts

@@ -9,9 +9,13 @@ import {
 import { database } from '../../../client'
 import {
   AI_OPERATION_ID_VALUE_GENERATE_OBJECT,
+  AI_OPERATION_ID_VALUE_GENERATE_OBJECT_DO_GENERATE,
   AI_OPERATION_ID_VALUE_GENERATE_TEXT,
+  AI_OPERATION_ID_VALUE_GENERATE_TEXT_DO_GENERATE,
   AI_OPERATION_ID_VALUE_STREAM_OBJECT,
+  AI_OPERATION_ID_VALUE_STREAM_OBJECT_DO_STREAM,
   AI_OPERATION_ID_VALUE_STREAM_TEXT,
+  AI_OPERATION_ID_VALUE_STREAM_TEXT_DO_STREAM,
   AI_OPERATION_ID_VALUE_TOOL,
   ATTR_AI_OPERATION_ID,
   ATTR_LATITUDE_INTERNAL,
@@ -142,6 +146,10 @@ export function extractSpanType(
     case AI_OPERATION_ID_VALUE_STREAM_TEXT:
     case AI_OPERATION_ID_VALUE_GENERATE_OBJECT:
     case AI_OPERATION_ID_VALUE_STREAM_OBJECT:
+    case AI_OPERATION_ID_VALUE_GENERATE_TEXT_DO_GENERATE:
+    case AI_OPERATION_ID_VALUE_STREAM_TEXT_DO_STREAM:
+    case AI_OPERATION_ID_VALUE_GENERATE_OBJECT_DO_GENERATE:
+    case AI_OPERATION_ID_VALUE_STREAM_OBJECT_DO_STREAM:
       return Result.ok(SpanType.Completion)
   }
 

packages/core/src/services/tracing/spans/processBulk.ts

@@ -16,9 +16,13 @@ import { cache as redis } from '../../../cache'
 import { database } from '../../../client'
 import {
   AI_OPERATION_ID_VALUE_GENERATE_OBJECT,
+  AI_OPERATION_ID_VALUE_GENERATE_OBJECT_DO_GENERATE,
   AI_OPERATION_ID_VALUE_GENERATE_TEXT,
+  AI_OPERATION_ID_VALUE_GENERATE_TEXT_DO_GENERATE,
   AI_OPERATION_ID_VALUE_STREAM_OBJECT,
+  AI_OPERATION_ID_VALUE_STREAM_OBJECT_DO_STREAM,
   AI_OPERATION_ID_VALUE_STREAM_TEXT,
+  AI_OPERATION_ID_VALUE_STREAM_TEXT_DO_STREAM,
   AI_OPERATION_ID_VALUE_TOOL,
   ATTR_AI_OPERATION_ID,
   ATTR_LATITUDE_TYPE,
@@ -579,6 +583,10 @@ export function extractSpanType(
     case AI_OPERATION_ID_VALUE_STREAM_TEXT:
     case AI_OPERATION_ID_VALUE_GENERATE_OBJECT:
     case AI_OPERATION_ID_VALUE_STREAM_OBJECT:
+    case AI_OPERATION_ID_VALUE_GENERATE_TEXT_DO_GENERATE:
+    case AI_OPERATION_ID_VALUE_STREAM_TEXT_DO_STREAM:
+    case AI_OPERATION_ID_VALUE_GENERATE_OBJECT_DO_GENERATE:
+    case AI_OPERATION_ID_VALUE_STREAM_OBJECT_DO_STREAM:
       return Result.ok(SpanType.Completion)
   }
 

packages/core/src/services/tracing/spans/prompt.ts

@@ -2,11 +2,17 @@ import { database } from '../../../client'
 import {
   ATTR_GEN_AI_REQUEST_PARAMETERS,
   ATTR_GEN_AI_REQUEST_TEMPLATE,
+  ATTR_LATITUDE_PROMPT_PATH,
+  HEAD_COMMIT,
   LogSources,
   SPAN_SPECIFICATIONS,
   SpanType,
 } from '../../../constants'
 import { Result } from '../../../lib/Result'
+import {
+  CommitsRepository,
+  DocumentVersionsRepository,
+} from '../../../repositories'
 import { SpanProcessArgs } from './shared'
 
 const specification = SPAN_SPECIFICATIONS[SpanType.Prompt]
@@ -16,8 +22,8 @@ export const PromptSpanSpecification = {
 }
 
 async function process(
-  { attributes }: SpanProcessArgs<SpanType.Prompt>,
-  _ = database,
+  { attributes, workspace }: SpanProcessArgs<SpanType.Prompt>,
+  db = database,
 ) {
   let parameters: Record<string, unknown>
   try {
@@ -28,17 +34,84 @@ async function process(
     parameters = {}
   }
 
-  return Result.ok({
+  // Get promptUuid from attributes, or try to resolve from promptPath
+  let promptUuid = attributes['latitude.documentUuid'] as string | undefined
+  const promptPath = attributes[ATTR_LATITUDE_PROMPT_PATH] as string | undefined
+  const projectId = attributes['latitude.projectId'] as number | undefined
+  const versionUuid =
+    (attributes['latitude.commitUuid'] as string) || HEAD_COMMIT
+
+  // If promptPath is provided but promptUuid is not, resolve it
+  if (promptPath && !promptUuid && projectId) {
+    const resolvedUuid = await resolvePromptPathToUuid({
+      promptPath,
+      projectId,
+      versionUuid,
+      workspaceId: workspace.id,
+      db,
+    })
+    if (resolvedUuid) promptUuid = resolvedUuid
+  }
+
+  const result = {
     parameters,
     template: attributes[ATTR_GEN_AI_REQUEST_TEMPLATE] as string,
     externalId: attributes['latitude.externalId'] as string,
 
     // References
     experimentUuid: attributes['latitude.experimentUuid'] as string,
-    promptUuid: attributes['latitude.documentUuid'] as string,
-    versionUuid: attributes['latitude.commitUuid'] as string,
+    promptUuid,
+    promptPath,
+    versionUuid,
     documentLogUuid: attributes['latitude.documentLogUuid'] as string,
-    projectId: attributes['latitude.projectId'] as number,
+    projectId,
     source: attributes['latitude.source'] as LogSources,
-  })
+  }
+
+  return Result.ok(result)
+}
+
+/**
+ * Resolves a prompt path to its document UUID by looking up the document
+ * in the specified project and version.
+ */
+async function resolvePromptPathToUuid({
+  promptPath,
+  projectId,
+  versionUuid,
+  workspaceId,
+  db,
+}: {
+  promptPath: string
+  projectId: number
+  versionUuid: string
+  workspaceId: number
+  db: typeof database
+}): Promise<string | undefined> {
+  try {
+    // Get the commit
+    const commitsRepo = new CommitsRepository(workspaceId, db)
+    const commitResult = await commitsRepo.getCommitByUuid({
+      uuid: versionUuid,
+      projectId,
+    })
+    if (commitResult.error) {
+      return undefined
+    }
+    const commit = commitResult.value
+
+    // Get the document by path
+    const docsRepo = new DocumentVersionsRepository(workspaceId, db)
+    const docResult = await docsRepo.getDocumentByPath({
+      commit,
+      path: promptPath,
+    })
+    if (docResult.error) {
+      return undefined
+    }
+
+    return docResult.value.documentUuid
+  } catch {
+    return undefined
+  }
 }

packages/telemetry/typescript/src/index.ts

@@ -1 +1,2 @@
+export * from './processors'
 export * from './sdk'

packages/telemetry/typescript/src/instrumentations/manual.ts

@@ -30,6 +30,7 @@ import {
   ATTR_HTTP_REQUEST_URL,
   ATTR_HTTP_RESPONSE_BODY,
   ATTR_HTTP_RESPONSE_HEADER,
+  ATTR_LATITUDE_PROMPT_PATH,
   ATTR_LATITUDE_TYPE,
   GEN_AI_TOOL_TYPE_VALUE_FUNCTION,
   HEAD_COMMIT,
@@ -125,11 +126,12 @@ export type EndHttpSpanOptions = EndSpanOptions & {
 export type PromptSpanOptions = StartSpanOptions & {
   documentLogUuid?: string // TODO(tracing): temporal related log, remove when observability is ready
   versionUuid?: string // Alias for commitUuid
-  promptUuid: string // Alias for documentUuid
+  promptUuid?: string // Alias for documentUuid
+  promptPath?: string // Path-based prompt identification (resolved server-side)
   projectId?: string
   experimentUuid?: string
   externalId?: string
-  template: string
+  template?: string
   parameters?: Record<string, unknown>
   source?: LogSources
 }
@@ -721,6 +723,7 @@ export class ManualInstrumentation implements BaseInstrumentation {
       documentLogUuid,
       versionUuid,
       promptUuid,
+      promptPath,
       projectId,
       experimentUuid,
       externalId,
@@ -739,19 +742,26 @@ export class ManualInstrumentation implements BaseInstrumentation {
     }
 
     const attributes = {
-      [ATTR_GEN_AI_REQUEST_TEMPLATE]: template,
+      ...(template && { [ATTR_GEN_AI_REQUEST_TEMPLATE]: template }),
       [ATTR_GEN_AI_REQUEST_PARAMETERS]: jsonParameters,
       ['latitude.commitUuid']: versionUuid || HEAD_COMMIT,
-      ['latitude.documentUuid']: promptUuid,
-      ['latitude.projectId']: projectId,
+      ...(promptUuid && { ['latitude.documentUuid']: promptUuid }),
+      ...(promptPath && { [ATTR_LATITUDE_PROMPT_PATH]: promptPath }),
+      ...(projectId && { ['latitude.projectId']: projectId }),
       ...(documentLogUuid && { ['latitude.documentLogUuid']: documentLogUuid }),
       ...(experimentUuid && { ['latitude.experimentUuid']: experimentUuid }),
       ...(externalId && { ['latitude.externalId']: externalId }),
       ...(source && { ['latitude.source']: source }),
       ...(rest.attributes || {}),
     }
 
-    return this.span(ctx, name || `prompt-${promptUuid}`, SpanType.Prompt, {
+    const spanName =
+      name ||
+      (promptPath
+        ? `prompt-${promptPath}`
+        : `prompt-${promptUuid || 'external'}`)
+
+    return this.span(ctx, spanName, SpanType.Prompt, {
       attributes,
     })
   }

packages/telemetry/typescript/src/processors/index.ts

@@ -0,0 +1 @@
+export { NormalizingSpanProcessor } from './normalize'