Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion apps/sim/providers/azure-openai/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,12 @@ describe('azureOpenAIProvider — SSRF pinning', () => {
expect('stream' in result).toBe(true)
if (!('stream' in result)) throw new Error('Expected streaming execution')
expect(result.execution.output.content).toBe('done')
expect(result.execution.output.tokens).toEqual({ input: 6, output: 3, total: 9 })
expect(result.execution.output.tokens).toEqual({
input: 6,
output: 3,
cacheRead: 0,
total: 9,
})
expect(result.execution.output.providerTiming?.iterations).toBe(2)
expect(
result.execution.output.providerTiming?.timeSegments?.filter(
Expand Down
60 changes: 14 additions & 46 deletions apps/sim/providers/azure-openai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { executeResponsesProviderRequest } from '@/providers/openai/core'
import { getChatCompletionConversationUsage } from '@/providers/openai-compat/conversation-usage'
import { createOpenAICompatibleAgentEventStream } from '@/providers/openai-compat/stream-events'
import { createChatCompletionUsageMeter } from '@/providers/openai-compat/usage-meter'
import { buildJsonSchemaResponseFormat } from '@/providers/response-format'
import { executeProviderTool } from '@/providers/runtime-context'
import { createSettledAgentEventStream } from '@/providers/stream-events'
Expand All @@ -62,7 +63,6 @@ import type {
} from '@/providers/types'
import { ProviderError } from '@/providers/types'
import {
calculateCost,
checkForForcedToolUsageOpenAI,
isFunctionToolCall,
prepareToolExecution,
Expand Down Expand Up @@ -244,22 +244,10 @@ async function executeChatCompletionsRequest(
request,
onComplete: ({ content, usage }) => {
output.content = content
output.tokens = {
input: usage.prompt_tokens,
output: usage.completion_tokens,
total: usage.total_tokens,
}

const costResult = calculateCost(
request.model,
usage.prompt_tokens,
usage.completion_tokens
)
output.cost = {
input: costResult.input,
output: costResult.output,
total: costResult.total,
}
const streamUsage = createChatCompletionUsageMeter(request.model)
streamUsage.record(usage)
output.tokens = { ...streamUsage.tokens }
output.cost = streamUsage.cost()

finalizeTiming()
},
Expand Down Expand Up @@ -289,11 +277,8 @@ async function executeChatCompletionsRequest(
const firstResponseTime = Date.now() - initialCallTime

let content = currentResponse.choices[0]?.message?.content || ''
const tokens = {
input: currentResponse.usage?.prompt_tokens || 0,
output: currentResponse.usage?.completion_tokens || 0,
total: currentResponse.usage?.total_tokens || 0,
}
const usageMeter = createChatCompletionUsageMeter(request.model)
usageMeter.record(currentResponse.usage)
const toolCalls: FunctionCallResponse[] = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
Expand Down Expand Up @@ -577,11 +562,7 @@ async function executeChatCompletionsRequest(
content = currentResponse.choices[0].message.content
}

if (currentResponse.usage) {
tokens.input += currentResponse.usage.prompt_tokens || 0
tokens.output += currentResponse.usage.completion_tokens || 0
tokens.total += currentResponse.usage.total_tokens || 0
}
usageMeter.record(currentResponse.usage)

iterationCount++
}
Expand Down Expand Up @@ -623,11 +604,7 @@ async function executeChatCompletionsRequest(
modelTime += synthesisEndTime - synthesisStartTime

content = synthesisResponse.choices[0]?.message?.content || content
if (synthesisResponse.usage) {
tokens.input += synthesisResponse.usage.prompt_tokens || 0
tokens.output += synthesisResponse.usage.completion_tokens || 0
tokens.total += synthesisResponse.usage.total_tokens || 0
}
usageMeter.record(synthesisResponse.usage)

enrichLastModelSegmentFromChatCompletions(
timeSegments,
Expand All @@ -640,8 +617,7 @@ async function executeChatCompletionsRequest(
if (request.stream) {
logger.info('Projecting settled response after tool processing')

const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
const toolCost = sumToolCosts(toolResults)
const accumulatedCost = usageMeter.cost(sumToolCosts(toolResults))

return createStreamingExecution({
model: request.model,
Expand All @@ -655,17 +631,8 @@ async function executeChatCompletionsRequest(
iterations: timeSegments.filter((segment) => segment.type === 'model').length,
timeSegments,
},
initialTokens: {
input: tokens.input,
output: tokens.output,
total: tokens.total,
},
initialCost: {
input: accumulatedCost.input,
output: accumulatedCost.output,
toolCost: toolCost || undefined,
total: accumulatedCost.total + toolCost,
},
initialTokens: { ...usageMeter.tokens },
initialCost: accumulatedCost,
toolCalls:
toolCalls.length > 0
? {
Expand All @@ -689,7 +656,8 @@ async function executeChatCompletionsRequest(
return {
content,
model: request.model,
tokens,
tokens: { ...usageMeter.tokens },
cost: usageMeter.cost(),
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
toolResults: toolResults.length > 0 ? toolResults : undefined,
timing: {
Expand Down
66 changes: 15 additions & 51 deletions apps/sim/providers/cerebras/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { createOpenAICompatAssistantHistory } from '@/providers/openai-compat/assistant-history'
import { getChatCompletionConversationUsage } from '@/providers/openai-compat/conversation-usage'
import { createOpenAICompatibleAgentEventStream } from '@/providers/openai-compat/stream-events'
import { createChatCompletionUsageMeter } from '@/providers/openai-compat/usage-meter'
import { buildJsonSchemaResponseFormat } from '@/providers/response-format'
import { executeProviderTool } from '@/providers/runtime-context'
import { createSettledAgentEventStream } from '@/providers/stream-events'
Expand All @@ -38,7 +39,6 @@ import type {
} from '@/providers/types'
import { ProviderError } from '@/providers/types'
import {
calculateCost,
isFunctionToolCall,
prepareToolExecution,
prepareToolsWithUsageControl,
Expand Down Expand Up @@ -160,22 +160,10 @@ export const cerebrasProvider: ProviderConfig = {
request,
onComplete: ({ content, usage }) => {
output.content = content
output.tokens = {
input: usage.prompt_tokens,
output: usage.completion_tokens,
total: usage.total_tokens,
}

const costResult = calculateCost(
request.model,
usage.prompt_tokens,
usage.completion_tokens
)
output.cost = {
input: costResult.input,
output: costResult.output,
total: costResult.total,
}
const streamUsage = createChatCompletionUsageMeter(request.model)
streamUsage.record(usage)
output.tokens = { ...streamUsage.tokens }
output.cost = streamUsage.cost()
},
}),
})
Expand All @@ -199,11 +187,8 @@ export const cerebrasProvider: ProviderConfig = {
const firstResponseTime = Date.now() - initialCallTime

let content = currentResponse.choices[0]?.message?.content || ''
const tokens = {
input: currentResponse.usage?.prompt_tokens || 0,
output: currentResponse.usage?.completion_tokens || 0,
total: currentResponse.usage?.total_tokens || 0,
}
const usageMeter = createChatCompletionUsageMeter(request.model)
usageMeter.record(currentResponse.usage)
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...formattedMessages]
Expand Down Expand Up @@ -476,11 +461,7 @@ export const cerebrasProvider: ProviderConfig = {
if (currentResponse.choices[0]?.message?.content) {
content = currentResponse.choices[0].message.content
}
if (currentResponse.usage) {
tokens.input += currentResponse.usage.prompt_tokens || 0
tokens.output += currentResponse.usage.completion_tokens || 0
tokens.total += currentResponse.usage.total_tokens || 0
}
usageMeter.record(currentResponse.usage)

enrichLastModelSegmentFromChatCompletions(
timeSegments,
Expand Down Expand Up @@ -525,11 +506,7 @@ export const cerebrasProvider: ProviderConfig = {
})

modelTime += thisModelTime
if (currentResponse.usage) {
tokens.input += currentResponse.usage.prompt_tokens || 0
tokens.output += currentResponse.usage.completion_tokens || 0
tokens.total += currentResponse.usage.total_tokens || 0
}
usageMeter.record(currentResponse.usage)

iterationCount++
}
Expand Down Expand Up @@ -577,11 +554,7 @@ export const cerebrasProvider: ProviderConfig = {
if (currentResponse.choices[0]?.message?.content) {
content = currentResponse.choices[0].message.content
}
if (currentResponse.usage) {
tokens.input += currentResponse.usage.prompt_tokens || 0
tokens.output += currentResponse.usage.completion_tokens || 0
tokens.total += currentResponse.usage.total_tokens || 0
}
usageMeter.record(currentResponse.usage)

enrichLastModelSegmentFromChatCompletions(
timeSegments,
Expand All @@ -604,8 +577,7 @@ export const cerebrasProvider: ProviderConfig = {
const totalDuration = providerEndTime - providerStartTime

if (request.stream) {
const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
const toolCost = sumToolCosts(toolResults)
const accumulatedCost = usageMeter.cost(sumToolCosts(toolResults))

const streamingResult = createStreamingExecution({
model: request.model,
Expand All @@ -619,17 +591,8 @@ export const cerebrasProvider: ProviderConfig = {
iterations: iterationCount + 1,
timeSegments,
},
initialTokens: {
input: tokens.input,
output: tokens.output,
total: tokens.total,
},
initialCost: {
input: accumulatedCost.input,
output: accumulatedCost.output,
toolCost: toolCost || undefined,
total: accumulatedCost.total + toolCost,
},
initialTokens: { ...usageMeter.tokens },
initialCost: accumulatedCost,
toolCalls:
toolCalls.length > 0
? {
Expand All @@ -652,7 +615,8 @@ export const cerebrasProvider: ProviderConfig = {
return {
content,
model: request.model,
tokens,
tokens: { ...usageMeter.tokens },
cost: usageMeter.cost(),
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
toolResults: toolResults.length > 0 ? toolResults : undefined,
timing: {
Expand Down
9 changes: 7 additions & 2 deletions apps/sim/providers/litellm/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ describe('litellmProvider.executeRequest', () => {
expect(payload.temperature).toBe(0.5)
expect(payload.max_completion_tokens).toBe(256)
expect(result.content).toBe('hello')
expect(result.tokens).toEqual({ input: 5, output: 3, total: 8 })
expect(result.tokens).toEqual({ input: 5, output: 3, cacheRead: 0, total: 8 })
})

it('forwards reasoning_effort only when set to a non-default value', async () => {
Expand Down Expand Up @@ -243,7 +243,12 @@ describe('litellmProvider.executeRequest', () => {

expect(mockCreate).toHaveBeenCalledTimes(2)
expect(result.execution.output.content).toBe('done')
expect(result.execution.output.tokens).toEqual({ input: 10, output: 6, total: 16 })
expect(result.execution.output.tokens).toEqual({
input: 10,
output: 6,
cacheRead: 0,
total: 16,
})
expect(result.execution.output.providerTiming?.iterations).toBe(2)
expect(
result.execution.output.providerTiming?.timeSegments?.filter(
Expand Down
Loading
Loading