| 1457 | } |
| 1458 | |
| 1459 | async onLLMEnd( |
| 1460 | returnIds: ICommonObject, |
| 1461 | output: string | ICommonObject, |
| 1462 | { model, provider }: { model?: string; provider?: string } = {} |
| 1463 | ) { |
| 1464 | // Extract content, usage metadata, and model name from output |
| 1465 | // Support both string (backward compatible) and structured object formats |
| 1466 | let outputText: string |
| 1467 | let usageMetadata: ICommonObject | undefined |
| 1468 | let modelName = model |
| 1469 | |
| 1470 | if (typeof output === 'string') { |
| 1471 | outputText = output |
| 1472 | } else { |
| 1473 | // Extract text content |
| 1474 | outputText = output.content ?? '' |
| 1475 | |
| 1476 | // Extract usage metadata (supports both LangChain and OpenAI field names) |
| 1477 | usageMetadata = output.usageMetadata ?? output.usage_metadata |
| 1478 | if (usageMetadata) { |
| 1479 | // Normalize field names for consistent access |
| 1480 | usageMetadata = { |
| 1481 | input_tokens: usageMetadata.input_tokens ?? usageMetadata.prompt_tokens, |
| 1482 | output_tokens: usageMetadata.output_tokens ?? usageMetadata.completion_tokens, |
| 1483 | total_tokens: usageMetadata.total_tokens |
| 1484 | } |
| 1485 | } |
| 1486 | |
| 1487 | // Extract model name from response metadata |
| 1488 | const responseMetadata = output.responseMetadata ?? output.response_metadata |
| 1489 | if (!model && responseMetadata) { |
| 1490 | modelName = responseMetadata.model ?? responseMetadata.model_name ?? responseMetadata.modelId |
| 1491 | } |
| 1492 | } |
| 1493 | |
| 1494 | if (Object.prototype.hasOwnProperty.call(this.handlers, 'langSmith')) { |
| 1495 | const llmRun: RunTree | undefined = this.handlers['langSmith'].llmRun[returnIds['langSmith'].llmRun] |
| 1496 | if (llmRun) { |
| 1497 | const outputs: ICommonObject = { |
| 1498 | generations: [outputText] |
| 1499 | } |
| 1500 | // Add usage_metadata and model info to run's extra.metadata for cost tracking |
| 1501 | // LangSmith requires: usage_metadata (with input_tokens, output_tokens, total_tokens) |
| 1502 | // and ls_model_name + ls_provider in metadata for automatic cost calculation |
| 1503 | if (usageMetadata || modelName) { |
| 1504 | if (!llmRun.extra) { |
| 1505 | llmRun.extra = {} |
| 1506 | } |
| 1507 | if (!llmRun.extra.metadata) { |
| 1508 | llmRun.extra.metadata = {} |
| 1509 | } |
| 1510 | if (usageMetadata) { |
| 1511 | llmRun.extra.metadata.usage_metadata = { |
| 1512 | input_tokens: usageMetadata.input_tokens, |
| 1513 | output_tokens: usageMetadata.output_tokens, |
| 1514 | total_tokens: usageMetadata.total_tokens |
| 1515 | } |
| 1516 | } |