From 68af8b3c189aa7f589f664b47f2c9a024cc3d54e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 21:16:25 -0800 Subject: [PATCH 01/28] fix(workflow-code): check lock parent cycles in linear time and cap lock entries at maxBlocks --- .../lib/api/contracts/workflow-source.test.ts | 28 ++++++++++ apps/sim/lib/api/contracts/workflow-source.ts | 19 ++++++- packages/workflow-code/src/lock.test.ts | 44 +++++++++++++++ packages/workflow-code/src/lock.ts | 56 ++++++++++++++----- 4 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 apps/sim/lib/api/contracts/workflow-source.test.ts diff --git a/apps/sim/lib/api/contracts/workflow-source.test.ts b/apps/sim/lib/api/contracts/workflow-source.test.ts new file mode 100644 index 00000000000..576267f654d --- /dev/null +++ b/apps/sim/lib/api/contracts/workflow-source.test.ts @@ -0,0 +1,28 @@ +import { SOURCE_LIMITS } from '@sim/workflow-code/limits' +import { describe, expect, it } from 'vitest' +import { workflowLockSchema } from '@/lib/api/contracts/workflow-source' + +function lockOf(blockCount: number, variableCount: number) { + const blocks: Record = {} + const variables: Record = {} + for (let i = 0; i < blockCount; i += 1) { + blocks[`b${i}`] = { id: `id${i}`, parent: null, ident: `b${i}`, order: i } + } + for (let i = 0; i < variableCount; i += 1) variables[`v${i}`] = `var${i}` + return { version: 1, blocks, variables } +} + +describe('workflowLockSchema', () => { + it('accepts a lock at the block and variable cap', () => { + const max = SOURCE_LIMITS.maxBlocks + expect(workflowLockSchema.safeParse(lockOf(max, max)).success).toBe(true) + }) + + it('refuses a lock with more blocks or variables than a workflow may hold', () => { + const over = SOURCE_LIMITS.maxBlocks + 1 + const blocks = workflowLockSchema.safeParse(lockOf(over, 0)) + const variables = workflowLockSchema.safeParse(lockOf(0, over)) + expect(blocks.error?.issues.map((issue) => issue.path.join('.'))).toEqual(['blocks']) + expect(variables.error?.issues.map((issue) => issue.path.join('.'))).toEqual(['variables']) + }) +}) diff --git a/apps/sim/lib/api/contracts/workflow-source.ts b/apps/sim/lib/api/contracts/workflow-source.ts index 720ad00ad6f..91e0bdadb4c 100644 --- a/apps/sim/lib/api/contracts/workflow-source.ts +++ b/apps/sim/lib/api/contracts/workflow-source.ts @@ -5,6 +5,7 @@ import type { WorkflowLockV1, } from '@sim/workflow-code' import { DIAGNOSTIC_CODES } from '@sim/workflow-code/diagnostics/codes' +import { SOURCE_LIMITS } from '@sim/workflow-code/limits' import { z } from 'zod' import { defineRouteContract } from '@/lib/api/contracts/types' import { workflowIdParamsSchema } from '@/lib/api/contracts/workflows' @@ -77,6 +78,11 @@ const lockLayoutSchema = withExtensions({ 'Stored canvas geometry; applied only to new blocks.' ) +/** A lock never binds more entries than a workflow may hold blocks (`parseWorkflowLock` agrees). */ +function atMostMaxBlocks(record: Record): boolean { + return Object.keys(record).length <= SOURCE_LIMITS.maxBlocks +} + const lockBlockSchema = withExtensions({ id: z.string().describe('Block id.'), parent: z.string().nullable().describe('Key of the enclosing loop or parallel, or null.'), @@ -107,8 +113,17 @@ const lockBlockSchema = withExtensions({ export const workflowLockSchema = withExtensions({ version: z.literal(1).describe('Lock format version.'), - blocks: z.record(z.string(), lockBlockSchema).describe('Block key → identity.'), - variables: z.record(z.string(), z.string()).describe('Variable key → variable id.'), + blocks: z + .record(z.string(), lockBlockSchema) + .refine(atMostMaxBlocks, `lock.blocks cannot hold more than ${SOURCE_LIMITS.maxBlocks} entries`) + .describe('Block key → identity.'), + variables: z + .record(z.string(), z.string()) + .refine( + atMostMaxBlocks, + `lock.variables cannot hold more than ${SOURCE_LIMITS.maxBlocks} entries` + ) + .describe('Variable key → variable id.'), } satisfies ShapeOf).describe( 'Identity sidecar (`.sim/lock.json`): binds the keys in the source to stored ids.' ) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index 54ba0becbb5..19eb6e2b5c7 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -1,4 +1,6 @@ +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' import { describe, expect, it } from 'vitest' +import { SOURCE_LIMITS } from './limits' import { parseSourceBase, parseWorkflowLock } from './lock' function validLock(): Record { @@ -175,6 +177,48 @@ describe('parseWorkflowLock', () => { ]) }) + it('reports only the blocks on a cycle, not the blocks that lead into one', () => { + const lock = validLock() + lock.blocks = { + a: { id: 'a', parent: 'b', ident: 'a', order: 0 }, + b: { id: 'b', parent: 'a', ident: 'b', order: 1 }, + c: { id: 'c', parent: 'a', ident: 'c', order: 2 }, + d: { id: 'd', parent: 'c', ident: 'd', order: 3 }, + } + expect(codesFor(lock)).toEqual([ + 'E_LOCK_PARENT /blocks/a/parent', + 'E_LOCK_PARENT /blocks/b/parent', + ]) + }) + + it('checks a long parent chain in linear time', () => { + expectLinearGrowth((n) => { + const blocks: Record = {} + for (let i = 0; i < n; i += 1) { + blocks[`b${i}`] = { + id: `id${i}`, + parent: i === 0 ? null : `b${i - 1}`, + ident: `b${i}`, + order: i, + } + } + const lock = { version: 1, blocks, variables: {} } + return () => parseWorkflowLock(lock) + }, SOURCE_LIMITS.maxBlocks / 8) + }) + + it('refuses more blocks or variables than a workflow may hold, before reading them', () => { + const over = SOURCE_LIMITS.maxBlocks + 1 + const blocks: Record = {} + const variables: Record = {} + for (let i = 0; i < over; i += 1) { + blocks[`b${i}`] = { id: `id${i}`, parent: null, ident: `b${i}`, order: i } + variables[`v${i}`] = `var${i}` + } + expect(codesFor({ version: 1, blocks, variables: {} })).toEqual(['E_LOCK_SHAPE /blocks']) + expect(codesFor({ version: 1, blocks: {}, variables })).toEqual(['E_LOCK_SHAPE /variables']) + }) + it('rejects branch keys that are not identifiers', () => { expect(codesFor(withBlock('route', { branches: { 'else-if-1': 'x' } }))).toEqual([ 'E_LOCK_BRANCH /blocks/route/branches/else-if-1', diff --git a/packages/workflow-code/src/lock.ts b/packages/workflow-code/src/lock.ts index 52b7d0ba015..6955918c01b 100644 --- a/packages/workflow-code/src/lock.ts +++ b/packages/workflow-code/src/lock.ts @@ -22,6 +22,7 @@ import { isIdentifierName } from './code-files/ref-path' import type { LockDiagnosticCode } from './diagnostics/codes' import { diagnostic } from './diagnostics/factory' import type { Diagnostic } from './diagnostics/frame' +import { SOURCE_LIMITS } from './limits' import type { BlockKey, BranchKey, Json, SubBlockId, VarKey } from './model' import { SDK_RESERVED_IDENTIFIERS } from './sdk' @@ -474,6 +475,7 @@ class LockReader { } } + const cyclic = blocksOnParentCycles(blocks) for (const [key, block] of Object.entries(blocks)) { if (block.parent === null) continue if (!Object.hasOwn(blocks, block.parent)) { @@ -482,22 +484,21 @@ class LockReader { ['blocks', key, 'parent'], `Parent "${block.parent}" is not a block in this lock.` ) - continue - } - const visited = new Set() - let current: BlockKey | null = block.parent - while (current !== null && Object.hasOwn(blocks, current) && !visited.has(current)) { - if (current === key) { - this.report('E_LOCK_PARENT', ['blocks', key, 'parent'], `"${key}" contains itself.`) - break - } - visited.add(current) - current = blocks[current].parent + } else if (cyclic.has(key)) { + this.report('E_LOCK_PARENT', ['blocks', key, 'parent'], `"${key}" contains itself.`) } } } + /** Whether `raw` holds more entries than a workflow may; reported so it is never read. */ + overCap(raw: unknown, at: readonly string[]): boolean { + if (!isRecordLike(raw) || Object.keys(raw).length <= SOURCE_LIMITS.maxBlocks) return false + this.report('E_LOCK_SHAPE', at, `Expected at most ${SOURCE_LIMITS.maxBlocks} entries.`) + return true + } + variables(raw: unknown): Record { + if (this.overCap(raw, ['variables'])) return {} const variables = this.idRecord(raw, ['variables']) for (const key of Object.keys(variables)) { if (!key || normalizeWorkflowBlockName(key) !== key) { @@ -512,6 +513,33 @@ class LockReader { } } +/** + * The blocks whose parent chain leads back to themselves. Each block is walked once: a walk stops + * at a block an earlier walk finished, and the blocks from where it meets its own path onward are + * the cycle. + */ +function blocksOnParentCycles(blocks: Record): Set { + const cyclic = new Set() + const finished = new Set() + for (const start of Object.keys(blocks)) { + const path: BlockKey[] = [] + const onPath = new Map() + let current: BlockKey | null = start + while (current !== null && Object.hasOwn(blocks, current) && !finished.has(current)) { + const index = onPath.get(current) + if (index !== undefined) { + for (const key of path.slice(index)) cyclic.add(key) + break + } + onPath.set(current, path.length) + path.push(current) + current = blocks[current].parent + } + for (const key of path) finished.add(key) + } + return cyclic +} + /** * A lock object of any version this build reads, as the current version (1) holds it, or `null` * for a version it cannot read. The one seam a lock format change goes through: v1 is current, @@ -604,13 +632,13 @@ export function parseWorkflowLock(input: unknown): ParseWorkflowLockResult { const extensions = reader.extensions(json, LOCK_FIELDS, []) const blockEntries: Array<[BlockKey, WorkflowLockBlock]> = [] - if (isRecordLike(json.blocks)) { + if (!isRecordLike(json.blocks)) { + reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') + } else if (!reader.overCap(json.blocks, ['blocks'])) { for (const [key, raw] of Object.entries(json.blocks)) { const block = reader.block(key, raw) if (block) blockEntries.push([key, block]) } - } else { - reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') } const blocks: Record = Object.fromEntries(blockEntries) reader.crossCheckBlocks(blocks) From caef3e475b9200c53844e6f5a8a4b77e7096d7d0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 21:25:52 -0800 Subject: [PATCH 02/28] refactor(utils): share one canonicalJson between table cells and cursor binding --- apps/sim/lib/api/cursor-binding.ts | 22 +-------------------- apps/sim/lib/sim-search/live/application.ts | 3 ++- apps/sim/lib/table/rows/cursor.ts | 4 ++-- apps/sim/lib/table/rows/live-schema.ts | 2 +- packages/table-cells/src/rows.ts | 17 ++-------------- packages/utils/src/object.ts | 16 +++++++++++++++ 6 files changed, 24 insertions(+), 40 deletions(-) diff --git a/apps/sim/lib/api/cursor-binding.ts b/apps/sim/lib/api/cursor-binding.ts index 920cc72886e..2bb07433332 100644 --- a/apps/sim/lib/api/cursor-binding.ts +++ b/apps/sim/lib/api/cursor-binding.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto' -import { filterUndefined } from '@sim/utils/object' +import { canonicalJson, filterUndefined } from '@sim/utils/object' /** * Caller-facing message for a cursor replayed under different filters. Separate @@ -124,26 +124,6 @@ export function instantScopePart(raw: string | undefined): string | undefined { return Number.isNaN(parsed) ? raw : new Date(parsed).toISOString() } -/** - * Deterministic JSON: object keys sorted so two structurally equal values - * serialize identically regardless of the key order they arrived in, and - * `undefined` members dropped so an omitted param and an absent one agree. - * - * Array order is preserved, because an array is a sequence in the general case. - * A filter whose array is really a set must canonicalize it first — see - * {@link parseUnorderedList} and {@link unorderedScopeOf} — or equivalent - * queries fingerprint differently and a valid cursor is refused. - */ -export function canonicalJson(value: unknown): string { - if (value instanceof Date) return JSON.stringify(value.toISOString()) - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` - const entries = Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(',')}}` -} - /** * Fingerprint of a canonical form, short enough to sit inside an opaque token. * diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 24bf5deaaf7..3623352902a 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -3,6 +3,7 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { mapWithConcurrency } from '@sim/utils/concurrency' +import { canonicalJson } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { z } from 'zod' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' @@ -16,7 +17,7 @@ import { nativeSearchQueriesSchema, workspaceSearchFiltersSchema, } from '@/lib/api/contracts/mothership-assistant-tools' -import { canonicalJson, fingerprint, instantScopePart } from '@/lib/api/cursor-binding' +import { fingerprint, instantScopePart } from '@/lib/api/cursor-binding' import { admitContent } from '@/lib/core/application' import { env } from '@/lib/core/config/env' import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index fab00d25925..4e579a5d8f9 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -18,8 +18,8 @@ * {@link assertCursorQueryBinding}. */ -import { isRecordLike } from '@sim/utils/object' -import { canonicalJson, canonicalUnorderedArray, fingerprint } from '@/lib/api/cursor-binding' +import { canonicalJson, isRecordLike } from '@sim/utils/object' +import { canonicalUnorderedArray, fingerprint } from '@/lib/api/cursor-binding' import { TableQueryValidationError } from '@/lib/table/errors' import type { Filter, Sort, TablePredicate, TableRow, TableRowsCursor } from '@/lib/table/types' diff --git a/apps/sim/lib/table/rows/live-schema.ts b/apps/sim/lib/table/rows/live-schema.ts index d27f7fcc183..9331931abe0 100644 --- a/apps/sim/lib/table/rows/live-schema.ts +++ b/apps/sim/lib/table/rows/live-schema.ts @@ -11,9 +11,9 @@ * write that waited sees the change. */ +import { canonicalJson } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { sql } from 'drizzle-orm' -import { canonicalJson } from '@/lib/api/cursor-binding' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { DbTransaction } from '@/lib/table/planner' diff --git a/packages/table-cells/src/rows.ts b/packages/table-cells/src/rows.ts index e3b15d5740a..b9a98420ddd 100644 --- a/packages/table-cells/src/rows.ts +++ b/packages/table-cells/src/rows.ts @@ -4,6 +4,7 @@ * batch that repeat an earlier row's unique value. The server's table writes and the CLI's offline * check of a workspace directory's rows both run these, so they agree on every row. */ +import { canonicalJson } from '@sim/utils/object' import { cellRuleOf } from './rules' import type { CellColumn, CellRow, CellValue } from './types' @@ -72,26 +73,12 @@ export function coerceRowCells( } } -/** - * Deterministic JSON: object keys sorted and `undefined` members dropped, so two structurally - * equal values serialize identically whatever their key order. - */ -function canonicalCellJson(value: unknown): string { - if (value instanceof Date) return JSON.stringify(value.toISOString()) - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(canonicalCellJson).join(',')}]` - const entries = Object.entries(value) - .filter(([, entry]) => entry !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalCellJson(entry)}`).join(',')}}` -} - /** * The key two unique-column values share when the unique check treats them as equal. Object keys * are sorted, since the check compares JSONB, where key order carries no meaning. */ export function uniqueValueKey(value: CellValue, column: CellColumn): string { - return canonicalCellJson(cellRuleOf(column).valueForEquality?.(value) ?? value) + return canonicalJson(cellRuleOf(column).valueForEquality?.(value) ?? value) } /** The errors of each refused row of a batch, by its index. */ diff --git a/packages/utils/src/object.ts b/packages/utils/src/object.ts index 20cb57c8779..195465ef93c 100644 --- a/packages/utils/src/object.ts +++ b/packages/utils/src/object.ts @@ -121,3 +121,19 @@ export function getValueAtPath(source: unknown, path: string): unknown { } return cursor } + +/** + * Deterministic JSON: object keys sorted so two structurally equal values serialize identically + * whatever order their keys arrived in, `undefined` members dropped so an omitted member and an + * absent one agree, and a `Date` written as its ISO string. Array order is kept: an array is a + * sequence, so a caller whose array is really a set sorts it first. + */ +export function canonicalJson(value: unknown): string { + if (value instanceof Date) return JSON.stringify(value.toISOString()) + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` + const entries = Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(',')}}` +} From 096e44600f5cfd9325d4af0b4d0c792735c32b0c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 20 Nov 2026 21:25:53 -0800 Subject: [PATCH 03/28] fix(workflow-code): infer renames from indexed candidates instead of every pair --- .../workflow-code/src/materialize/identity.ts | 312 +++++++++++++++--- .../src/materialize/scaling.test.ts | 109 ++++++ 2 files changed, 366 insertions(+), 55 deletions(-) create mode 100644 packages/workflow-code/src/materialize/scaling.test.ts diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 7d325c35052..853cbe696fd 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -1,3 +1,4 @@ +import { canonicalJson } from '@sim/utils/object' import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references' import { type BlockState, normalizeWorkflowBlockName } from '@sim/workflow-types/workflow' import type { CanonicalizeResult } from '../canonicalize' @@ -156,11 +157,11 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { inputs.compiledIdents ? getOwn(inputs.compiledIdents, key) : undefined const resolved = new Map() - /** The ids `resolved` holds, so a lock id's reuse check is one lookup. */ - const resolvedIds = new Set() + /** The key `resolved` holds each id under, so finding an id's block is one lookup. */ + const keyOfId = new Map() const resolve = (key: BlockKey, block: ResolvedBlock) => { resolved.set(key, block) - resolvedIds.add(block.id) + keyOfId.set(block.id, key) } const claimed = new Set() /** Keys that end as created blocks no later rule may match (retype, or a lock id to reuse). */ @@ -258,6 +259,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { workflow, prev, resolved, + keyOfId, claimed, open, match, @@ -278,7 +280,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { lockId !== undefined && !claimed.has(lockId) && !Object.hasOwn(prevBlocks, lockId) && - !resolvedIds.has(lockId) + !keyOfId.has(lockId) if (lockId !== undefined && !reusable) { diagnostics.push( diagnostic( @@ -341,6 +343,7 @@ interface InferenceState { workflow: CanonicalWorkflow prev: PrevGraph resolved: Map + keyOfId: ReadonlyMap claimed: Set open: () => BlockKey[] match: (key: BlockKey, prevId: string, via: IdentityMatchRule) => boolean @@ -354,14 +357,31 @@ interface InferenceState { catalog: BlockCatalog } +type CanonicalEdges = CanonicalWorkflow['edges'] + +/** Each block's edges, in or out, so a block's neighbourhood reads only its own edges. */ +function edgesByEndpoint(edges: CanonicalEdges): Map { + const byKey = new Map() + const add = (key: BlockKey, edge: CanonicalEdges[number]) => { + const list = byKey.get(key) + if (list) list.push(edge) + else byKey.set(key, [edge]) + } + for (const edge of edges) { + add(edge.from, edge) + if (edge.to !== edge.from) add(edge.to, edge) + } + return byKey +} + /** `in:` / `out:` for every already-bound neighbour of a block. */ function neighbourhood( - edges: CanonicalWorkflow['edges'], + edgesAt: ReadonlyMap, key: BlockKey, idOf: (key: BlockKey) => string | undefined ): Set { const result = new Set() - for (const edge of edges) { + for (const edge of edgesAt.get(key) ?? []) { if (edge.from === key) { const id = idOf(edge.to) if (id !== undefined) result.add(`out:${id}`) @@ -378,17 +398,16 @@ function sameSet(left: Set, right: Set): boolean { return left.size === right.size && [...left].every((item) => right.has(item)) } -/** A block's params with every `` reference token rewritten to `to`. */ -function withKeyRenamed(params: CanonicalBlock['params'], from: BlockKey, to: BlockKey) { - const rewrite = (value: unknown): unknown => { - if (typeof value === 'string') { - return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) - } - if (Array.isArray(value)) return value.map(rewrite) - if (value === null || typeof value !== 'object') return value - return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewrite(item)])) +/** `value` with every `` reference token rewritten to `to`. */ +function withKeyRenamed(value: unknown, from: BlockKey, to: BlockKey): unknown { + if (typeof value === 'string') { + return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) } - return rewrite(params) + if (Array.isArray(value)) return value.map((item) => withKeyRenamed(item, from, to)) + if (value === null || typeof value !== 'object') return value + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, withKeyRenamed(item, from, to)]) + ) } /** @@ -402,10 +421,13 @@ function settingsAgree( key: BlockKey, prevKey: BlockKey ): boolean { - const left = target.params - const right = withKeyRenamed(prev.params, prevKey, key) as CanonicalBlock['params'] - const shared = Object.keys(left).filter((id) => Object.hasOwn(right, id)) - return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) + let shared = false + for (const [id, value] of Object.entries(target.params)) { + if (!Object.hasOwn(prev.params, id)) continue + if (!deepEqual(value, withKeyRenamed(prev.params[id], prevKey, key))) return false + shared = true + } + return shared } /** Every string under `value`, depth first. */ @@ -418,21 +440,93 @@ function* strings(value: unknown): Generator { } /** - * Whether a block holds a reference to `key` in any field a reference can sit in (its params, a - * condition or router's branches, a loop or parallel's collection), its head read as the resolver - * reads it: normalized, so `` references `fetch`. + * The keys a block references in any field a reference can sit in (its params, a condition or + * router's branches, a loop or parallel's collection), each head read as the resolver reads it: + * normalized, so `` references `fetch`. */ -function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { +function referencedKeys(block: CanonicalBlock): Set { + const keys = new Set() for (const text of strings([block.params, block.branches, block.subflow])) { for (const token of findWorkflowReferenceTokens(text)) { if (token.kind !== 'workflow') continue const head = parseStoredPath(token.value.slice(1, -1))?.[0] - if (head?.kind === 'name' && normalizeWorkflowBlockName(head.name) === key) return true + if (head?.kind === 'name') keys.add(normalizeWorkflowBlockName(head.name)) } } + return keys +} + +/** Whether any string under `value` holds a `<`, the only way a reference can sit in it. */ +function holdsAngle(value: unknown): boolean { + for (const text of strings(value)) if (text.includes('<')) return true return false } +/** + * The blocks among `blocks` whose settings may agree with a given block's ({@link settingsAgree}), + * a superset found without comparing every pair, and the same whichever side is indexed. Settings + * agree only when every param both blocks hold is equal, so a match either lacks a param the given + * block holds or holds it equal. A value with no `<` holds no reference, so the old-key rewrite + * leaves it as it is and equal values have equal canonical JSON; a value with one keeps one after + * the rewrite, so it can only equal another such value. + */ +function settingsIndex(blocks: ReadonlyArray<[BlockKey, CanonicalBlock]>) { + const byValue = new Map() + const holders = new Map>() + const bucketOf = (id: string, value: unknown) => + holdsAngle(value) ? `<\u0000${id}` : `=\u0000${id}\u0000${canonicalJson(value)}` + for (const [key, block] of blocks) { + for (const [id, value] of Object.entries(block.params)) { + const bucket = bucketOf(id, value) + const keys = byValue.get(bucket) + if (keys) keys.push(key) + else byValue.set(bucket, [key]) + const held = holders.get(id) + if (held) held.add(key) + else holders.set(id, new Set([key])) + } + } + return function* matches(block: CanonicalBlock): Generator { + let best: { id: string; equal: BlockKey[] } | undefined + let bestSize = Number.POSITIVE_INFINITY + for (const [id, value] of Object.entries(block.params)) { + const equal = byValue.get(bucketOf(id, value)) ?? [] + const size = blocks.length - (holders.get(id)?.size ?? 0) + equal.length + if (size < bestSize) { + best = { id, equal } + bestSize = size + } + } + if (!best) return + const held = holders.get(best.id) + for (const [key] of blocks) if (!held?.has(key)) yield key + yield* best.equal + } +} + +/** + * The first half (rounded up) of a block's bound neighbours in `order`. Two blocks share a + * majority of their neighbours, the `connections` signal, only if these halves meet: the earliest + * shared neighbour sits in both, since more than half of each side's neighbours are shared and + * come at or after it. + */ +function neighbourPrefix(items: Set, order: (left: string, right: string) => number) { + return [...items].sort(order).slice(0, Math.ceil(items.size / 2)) +} + +/** Each key's items, inverted: item → the keys that hold it. */ +function invert(entries: Iterable<[BlockKey, Iterable]>): Map { + const byItem = new Map() + for (const [key, items] of entries) { + for (const item of items) { + const keys = byItem.get(item) + if (keys) keys.push(key) + else byItem.set(item, [key]) + } + } + return byItem +} + /** * Whether a block reads variable `key`: a `` reference in any field a reference can * sit in, or a variable node (a Variables block's assignment target). @@ -553,10 +647,41 @@ function inferRenames(state: InferenceState): void { return true } - const neighbours = (key: BlockKey, prevKey: BlockKey) => ({ - target: neighbourhood(workflow.edges, key, idOfTarget), - prev: neighbourhood(prevCanonical.edges, prevKey, idOfPrev), - }) + const targetEdges = edgesByEndpoint(workflow.edges) + const prevEdges = edgesByEndpoint(prevCanonical.edges) + const targetSide = (key: BlockKey) => neighbourhood(targetEdges, key, idOfTarget) + const prevSide = (prevKey: BlockKey) => neighbourhood(prevEdges, prevKey, idOfPrev) + + /** Each stored block's referrers: the other stored blocks that reference its key. */ + const referrersOf = new Map() + for (const [other, block] of Object.entries(prevCanonical.blocks)) { + for (const key of referencedKeys(block)) { + if (key === other || !Object.hasOwn(prevCanonical.blocks, key)) continue + const referrers = referrersOf.get(key) + if (referrers) referrers.push(other) + else referrersOf.set(key, [other]) + } + } + const targetReferences = new Map>() + const referencedByTarget = (key: BlockKey): Set => { + let keys = targetReferences.get(key) + if (!keys) { + keys = referencedKeys(workflow.blocks[key]) + targetReferences.set(key, keys) + } + return keys + } + /** The blocks that now stand where `prevKey`'s referrers stood, or none when one is unbound. */ + const renamedReferrers = (prevKey: BlockKey): BlockKey[] => { + const now: BlockKey[] = [] + for (const other of referrersOf.get(prevKey) ?? []) { + const id = idOfPrev(other) + const key = id === undefined ? undefined : state.keyOfId.get(id) + if (key === undefined) return [] + now.push(key) + } + return now + } const byConnections = (): boolean => { let progressed = false @@ -564,8 +689,7 @@ function inferRenames(state: InferenceState): void { if (targets.length !== 1 || prevKeys.length !== 1) continue const [key] = targets const [prevKey] = prevKeys - const sides = neighbours(key, prevKey) - if (!sameSet(sides.target, sides.prev)) continue + if (!sameSet(targetSide(key), prevSide(prevKey))) continue const triggerHeld = isTriggerLike(workflow.blocks[key], catalog) && !settingsAgree(workflow.blocks[key], prevCanonical.blocks[prevKey], key, prevKey) @@ -574,47 +698,125 @@ function inferRenames(state: InferenceState): void { return progressed } + /** Bound neighbourhoods of one group's blocks, read once per group. */ + interface GroupSides { + targets: Map> + vacated: Map> + } + /** The strong signal tying new block `key` to vacated stored block `prevKey`, if any. */ - const signal = (key: BlockKey, prevKey: BlockKey): InferenceBasis | null => { + const signal = (key: BlockKey, prevKey: BlockKey, sides: GroupSides): InferenceBasis | null => { const target = workflow.blocks[key] if (settingsAgree(target, prevCanonical.blocks[prevKey], key, prevKey)) return 'settings' if (isTriggerLike(target, catalog)) return null - const sides = neighbours(key, prevKey) - const shared = [...sides.target].filter((item) => sides.prev.has(item)).length - const union = new Set([...sides.target, ...sides.prev]).size + const targetItems = sides.targets.get(key) ?? new Set() + const prevItems = sides.vacated.get(prevKey) ?? new Set() + const shared = [...targetItems].filter((item) => prevItems.has(item)).length + const union = targetItems.size + prevItems.size - shared if (shared > 0 && shared * 2 > union) return 'connections' - const referrers = Object.keys(prevCanonical.blocks).filter( - (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other], prevKey) - ) + const referrers = renamedReferrers(prevKey) const renamedAtEverySite = - referrers.length > 0 && - referrers.every((other) => { - const id = idOfPrev(other) - const now = id === undefined ? undefined : [...resolved].find(([, b]) => b.id === id)?.[0] - return now !== undefined && referencesKey(workflow.blocks[now], key) - }) + referrers.length > 0 && referrers.every((now) => referencedByTarget(now).has(key)) return renamedAtEverySite ? 'references' : null } + /** + * The pairs of one group that may show a signal, found from either side without judging every + * pair: equal settings ({@link settingsIndex}), shared neighbours ({@link neighbourPrefix}), or + * every referrer now referencing the new block. A superset, so {@link signal} need judge only + * these. + */ + const groupSearch = (sides: GroupSides) => { + const blocksOf = (keys: Iterable, from: CanonicalWorkflow) => + [...keys].map((key): [BlockKey, CanonicalBlock] => [key, from.blocks[key]]) + const vacatedBySettings = settingsIndex(blocksOf(sides.vacated.keys(), prevCanonical)) + const targetsBySettings = settingsIndex(blocksOf(sides.targets.keys(), workflow)) + + const frequency = new Map() + for (const items of [...sides.targets.values(), ...sides.vacated.values()]) { + for (const item of items) frequency.set(item, (frequency.get(item) ?? 0) + 1) + } + const order = (left: string, right: string) => + (frequency.get(left) ?? 0) - (frequency.get(right) ?? 0) || (left < right ? -1 : 1) + const prefixes = (side: Map>) => + new Map([...side].map(([key, items]) => [key, neighbourPrefix(items, order)])) + const targetPrefixes = prefixes(sides.targets) + const vacatedPrefixes = prefixes(sides.vacated) + const vacatedByNeighbour = invert(vacatedPrefixes) + const targetsByNeighbour = invert(targetPrefixes) + + const referenced = new Map() + for (const prevKey of sides.vacated.keys()) { + const [first, ...rest] = renamedReferrers(prevKey).map(referencedByTarget) + referenced.set( + prevKey, + [...(first ?? [])].filter((key) => rest.every((keys) => keys.has(key))) + ) + } + const referencing = invert(referenced) + + function* unique(lists: Iterable>): Generator { + const seen = new Set() + for (const list of lists) { + for (const key of list) { + if (seen.has(key)) continue + seen.add(key) + yield key + } + } + } + return { + vacatedFor: (key: BlockKey) => + unique([ + vacatedBySettings(workflow.blocks[key]), + ...(targetPrefixes.get(key) ?? []).map((item) => vacatedByNeighbour.get(item) ?? []), + referencing.get(key) ?? [], + ]), + targetsFor: (prevKey: BlockKey) => + unique([ + targetsBySettings(prevCanonical.blocks[prevKey]), + ...(vacatedPrefixes.get(prevKey) ?? []).map((item) => targetsByNeighbour.get(item) ?? []), + referenced.get(prevKey) ?? [], + ]), + } + } + + /** + * Pairs each new block that shows a signal for exactly one vacated block, which no other new + * block shows one for. A new block stops at its second signal, since it can no longer pair; a + * vacated block's rivals are then sought only for the new blocks that would pair with it. + */ const byVacatedLock = (): boolean => { let progressed = false for (const { targets, prevKeys } of groups()) { const vacated = prevKeys.filter((prevKey) => lockedIds.has(prevIdentity[prevKey].id)) if (vacated.length === 0) continue - const signals = new Map>() + const sides: GroupSides = { + targets: new Map(targets.map((key) => [key, targetSide(key)])), + vacated: new Map(vacated.map((prevKey) => [prevKey, prevSide(prevKey)])), + } + const search = groupSearch(sides) + const chosen: Array<[BlockKey, BlockKey, InferenceBasis]> = [] for (const key of targets) { - const found = new Map() - for (const prevKey of vacated) { - const basis = signal(key, prevKey) - if (basis) found.set(prevKey, basis) + const found: Array<[BlockKey, InferenceBasis]> = [] + for (const prevKey of search.vacatedFor(key)) { + const basis = signal(key, prevKey, sides) + if (basis) found.push([prevKey, basis]) + if (found.length > 1) break + } + if (found.length !== 1) continue + const [[prevKey, basis]] = found + let rivalled = false + for (const other of search.targetsFor(prevKey)) { + if (other !== key && signal(other, prevKey, sides) !== null) { + rivalled = true + break + } } - signals.set(key, found) + if (!rivalled) chosen.push([key, prevKey, basis]) } - for (const [key, found] of signals) { - if (found.size !== 1) continue - const [[prevKey, basis]] = [...found] - const rivals = [...signals.values()].filter((other) => other.has(prevKey)).length - if (rivals === 1 && pair(key, prevKey, basis)) progressed = true + for (const [key, prevKey, basis] of chosen) { + if (pair(key, prevKey, basis)) progressed = true } } return progressed diff --git a/packages/workflow-code/src/materialize/scaling.test.ts b/packages/workflow-code/src/materialize/scaling.test.ts new file mode 100644 index 00000000000..29c728b8f3a --- /dev/null +++ b/packages/workflow-code/src/materialize/scaling.test.ts @@ -0,0 +1,109 @@ +/** + * Rename inference (identity rule 4) grows linearly with the number of renamed blocks (see + * `expectLinearGrowth` for how a case is measured). + */ + +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' +import { describe, expect, it } from 'vitest' +import type { CanonicalBlock } from '../model' +import { + blockState, + CATALOG, + canonical, + edge, + fixtureHooks, + lockFor, + RESOURCES, + workflowState, +} from './fixtures.test-helpers' +import { materialize } from './index' + +interface RenameShape { + /** The readers now read the new blocks, so each pairs by its renamed references. */ + renameReaders?: boolean + /** Every function block hangs off one hub, so every pair shares its one neighbour. */ + hub?: boolean +} + +/** + * `n` function blocks, each read by its own api block, replaced by `n` new function blocks under + * new keys, idents and code. Without a signal ({@link RenameShape}) each new block is created. + */ +function renameAll(n: number, { renameReaders = false, hub = false }: RenameShape) { + const prev = workflowState( + [ + blockState('hub', 'api', 'Hub', { url: 'https://hub' }), + ...Array.from({ length: n }, (_, i) => [ + blockState(`fn${i}`, 'function', `Old${i}`, { + language: 'javascript', + code: `return ${i}`, + }), + blockState(`rd${i}`, 'api', `Reader${i}`, { url: `https://x/` }), + ]).flat(), + ], + { edges: hub ? Array.from({ length: n }, (_, i) => edge(`e${i}`, 'hub', `fn${i}`)) : [] } + ) + const lock = lockFor(prev) + const prevCanonical = canonical(prev, lock) + const blocks: Record = {} + for (const [key, block] of Object.entries(prevCanonical.workflow.blocks)) { + if (block.type === 'function') { + const index = key.slice('old'.length) + const next = `new${index}` + blocks[next] = { + ...block, + key: next, + name: `New${index}`, + params: { ...block.params, code: { kind: 'value', value: `return '${next}'` } }, + } + } else { + const params = JSON.stringify(block.params) + blocks[key] = { + ...block, + params: JSON.parse(renameReaders ? params.replace(/ ({ + ...link, + to: link.to.replace(/^old/, 'new'), + })) + const workflow = { ...prevCanonical.workflow, blocks, edges } + const compiledIdents = Object.fromEntries(Object.keys(blocks).map((key) => [key, `x_${key}`])) + return () => + materialize(workflow, lock, { + catalog: CATALOG, + hooks: fixtureHooks(), + resources: RESOURCES, + prev, + prevCanonical, + lockFreshness: 'fresh', + compiledIdents, + }) +} + +describe('materialize scaling', () => { + it('pairs blocks renamed with their references, and nothing else', () => { + const viaReferences = renameAll(3, { renameReaders: true })().entries + expect(viaReferences.filter((entry) => entry.status === 'created')).toEqual([]) + expect(viaReferences).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'new1', id: 'fn1', via: 'inferred' }) + ) + for (const shape of [{}, { hub: true }]) { + const unpaired = renameAll(3, shape)().entries + expect(unpaired.filter((entry) => entry.status === 'created')).toHaveLength(3) + } + }) + + it('infers renames in linear time when no new block shows a signal', () => { + expectLinearGrowth((n) => renameAll(n, {}), 16) + }) + + it('infers renames in linear time when every new block pairs by its references', () => { + expectLinearGrowth((n) => renameAll(n, { renameReaders: true }), 16) + }) + + it('infers renames in linear time when every pair shares its one neighbour', () => { + expectLinearGrowth((n) => renameAll(n, { hub: true }), 16) + }) +}) From 1f0763088cc1b5864a3d892ee4425cf43f4a5784 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 00:50:28 -0800 Subject: [PATCH 04/28] fix(workflow-code): defer did-you-mean hints past the caps and refuse a literal tag naming no block A capped W_ENV_UNKNOWN or former-block W_UNRESOLVED_REF no longer computes its hint. A literal whose head names no block (and no renamed block) is W_UNRESOLVED_REF, refused on a changed block as E_APPLY_UNRESOLVED_REF, matching the server lint's E_APPLY_LINT. --- apps/docs/content/docs/cli/reference.mdx | 4 +- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- packages/workflow-code/src/compile/context.ts | 10 +++ .../src/compile/ergonomics.test.ts | 14 ++-- packages/workflow-code/src/compile/index.ts | 3 + .../workflow-code/src/compile/scaling.test.ts | 55 ++++++++++++- packages/workflow-code/src/compile/values.ts | 81 ++++++++++++++----- .../src/diagnostics/refusals.test.ts | 28 +++++++ .../workflow-code/src/diagnostics/refusals.ts | 6 +- 10 files changed, 174 insertions(+), 31 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index b91e32700b9..e4f0517f93b 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -10018,7 +10018,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 7e76fc8249d..76c3095768b 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 64e39c047d7..c91beb1ebea 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -787,7 +787,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index fbd8f89b3df..54ad9191a93 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -180,6 +180,8 @@ export interface CompileContext { * written last (`checkDynamicOutputRefs`). */ dynamicOutputRefs: DynamicOutputRef[] + /** Literal tags whose head names no block, checked once renames are known (`checkUnresolvedTags`). */ + unresolvedTags: LiteralTag[] /** `x.error` references, checked once the edges are known (`checkErrorOutputRefs`). */ errorOutputRefs: Array> /** @@ -194,6 +196,14 @@ export interface CompileContext { valueConsts: Set } +/** A `` in literal text whose head is no declared block's key. */ +export interface LiteralTag { + token: string + normalized: BlockKey + at: Range + blockKey: BlockKey +} + /** A reference path into a block whose outputs its `responseFormat` declares. */ interface DynamicOutputRef { at: Range diff --git a/packages/workflow-code/src/compile/ergonomics.test.ts b/packages/workflow-code/src/compile/ergonomics.test.ts index 73f32f8eee1..61efa0c2b41 100644 --- a/packages/workflow-code/src/compile/ergonomics.test.ts +++ b/packages/workflow-code/src/compile/ergonomics.test.ts @@ -476,14 +476,18 @@ describe('header, names and hand-written references', () => { }) it.each([ - ['a cased block name', "''", true], - ['a name that is no block', "''", true], - ['the normalized key of a block', "''", false], - ])('warns W_REF_NOT_NORMALIZED for %s: %s', (_label, literal, warns) => { + ['a cased block name', "''", 'W_REF_NOT_NORMALIZED'], + ['a name that is no block', "''", 'W_UNRESOLVED_REF'], + ['a bare tag that is no reference', "''", 'W_REF_NOT_NORMALIZED'], + ['the normalized key of a block', "''", undefined], + ])('warns for %s: %s', (_label, literal, warning) => { const result = run( source(` const a = api('A', { url: 'x' })\n api('B', { url: ${literal} })`) ) - expect(codes(result).includes('W_REF_NOT_NORMALIZED')).toBe(warns) + const found = codes(result).filter( + (code) => code === 'W_REF_NOT_NORMALIZED' || code === 'W_UNRESOLVED_REF' + ) + expect(found).toEqual(warning ? [warning] : []) if (literal === "''") { const warning = result.diagnostics.find((d) => d.code === 'W_REF_NOT_NORMALIZED') expect(warning?.hint).toContain('${a.data}') diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index dce5a31729c..cf7336bedd1 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -42,6 +42,7 @@ import { checkCodeFileRefs, checkDynamicOutputRefs, checkErrorOutputRefs, + checkUnresolvedTags, compileJsonLiteral, objectEntries, } from './values' @@ -574,6 +575,7 @@ function compileEntry( envReferences: new Set(), jsonFields: [], dynamicOutputRefs: [], + unresolvedTags: [], errorOutputRefs: [], refusedBranches: new Map(), valueConsts: new Set(), @@ -692,6 +694,7 @@ function compileEntry( : new Map() const renamed = new Map([...unreadRenames, ...(input.renamedKeys ?? [])]) checkCodeFileRefs(ctx, renamed) + checkUnresolvedTags(ctx, renamed) const stored = storedView(input.stored, renamed) const refTypes: RefTypeScope = { diff --git a/packages/workflow-code/src/compile/scaling.test.ts b/packages/workflow-code/src/compile/scaling.test.ts index 409ec37c78f..e0b62ea33cf 100644 --- a/packages/workflow-code/src/compile/scaling.test.ts +++ b/packages/workflow-code/src/compile/scaling.test.ts @@ -3,8 +3,45 @@ */ import { expectLinearGrowth } from '@sim/testing/helpers/scaling' -import { describe, expect, it } from 'vitest' +import { describe, it } from 'vitest' +import { SOURCE_LIMITS } from '../limits' +import { TEST_CATALOG } from './catalog.test-helpers' import { stronglyConnected } from './edges' +import { compileWorkflow } from './index' + +const ENTRY = 'workflows/test.ts' + +/** + * `n` api blocks and a function whose code file reads `refs` references `reference(i)` writes, + * nearly all of them findings past the warning caps. + */ +function compileReading( + n: number, + refs: number, + reference: (i: number) => string, + context: { knownKeys?: ReadonlySet; envNames?: ReadonlySet } +) { + const blocks = Array.from({ length: n }, (_, i) => ` const a${i} = api('A${i}', { url: 'x' })`) + const text = [ + '// @sim-sdk 1', + "import { api, codeFile, fn, trigger, workflow } from '@sim/workflow-sdk'", + "export default workflow('Test', {}, () => {", + " const start = trigger.start('Start', {})", + ...blocks, + " const f = fn('F', { code: codeFile('./functions/f.js') })", + '})', + '', + ].join('\n') + const code = Array.from({ length: refs }, (_, i) => `${reference(i)}\n`).join('') + return () => + compileWorkflow({ + entry: { path: ENTRY, text }, + files: new Map([['workflows/functions/f.js', code]]), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...context, + }) +} describe('compile scaling', () => { it('finds the cycles of a block wired to many targets in linear time', () => { @@ -14,4 +51,20 @@ describe('compile scaling', () => { return () => stronglyConnected(keys, (key) => (key === 'hub' ? [...targets] : [])) }, 5_000) }) + + it('computes no former-block hint for a reference past the caps', () => { + // Many references per block, so the hint's scan of every block outweighs compiling them. + const knownKeys = new Set(['start', 'gone']) + expectLinearGrowth( + (n) => compileReading(n, n * 80, () => '$ref.gone.x', { knownKeys }), + Math.floor((SOURCE_LIMITS.maxBlocks - 40) / 8) + ) + }) + + it('computes no did-you-mean for an env name past the caps', () => { + expectLinearGrowth((n) => { + const envNames = new Set(Array.from({ length: n }, (_, i) => `KNOWN_${i}`)) + return compileReading(n, n, (i) => `$env.MISSING_${i}`, { envNames }) + }, 200) + }) }) diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index 86c06ecc451..67976378dd3 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -57,6 +57,7 @@ import { containerKindOf, enclosingContainers, type JsonFieldEntry, + type LiteralTag, lookup, rangeOf, reportNotImported, @@ -247,9 +248,11 @@ function checkIterationTag( } /** - * `W_REF_NOT_NORMALIZED` for each hand-written `` in literal text whose head is not a - * normalized block key, or names no declared block. Such text is kept verbatim, so a rename in the - * same apply never rewrites it; the block's ident is the reference that follows the block. + * Checks each hand-written `` in literal text whose head is not a normalized block key, + * or names no declared block. Such text is kept verbatim, so a rename in the same apply never + * rewrites it; the block's ident is the reference that follows the block. A head written other + * than as its block's key is `W_REF_NOT_NORMALIZED`. A path whose head names no block at all + * waits for {@link checkUnresolvedTags}, which knows the apply's renames. */ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: ValueSite): void { // A note never runs, so nothing in it resolves: `` in its markdown is text, not a reference. @@ -267,18 +270,53 @@ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: V checkIterationTag(ctx, rangeOf(ctx, node), site.decl.key, segments) continue } - const target = ctx.declsByKey.get(normalized) + const tag = { token: token.value, normalized, at: rangeOf(ctx, node), blockKey: site.decl.key } + if (!ctx.declsByKey.has(normalized) && segments.length > 1) ctx.unresolvedTags.push(tag) + else reportNotNormalized(ctx, tag) + } +} + +function reportNotNormalized(ctx: CompileContext, tag: LiteralTag): void { + const target = ctx.declsByKey.get(tag.normalized) + const segments = parseStoredPath(tag.token.slice(1, -1)) ?? [] + ctx.reporter.report( + 'W_REF_NOT_NORMALIZED', + target + ? `${tag.token} is not written with the block key ${tag.normalized}; the text is kept as written.` + : `${tag.token} names no block; the text is kept as written.`, + tag.at, + { + hint: target?.ident + ? `Write \${${formatMemberPath(target.ident, segments.slice(1))}} in a template, so a rename follows the block.` + : 'Reference a declared block through its ident, so a rename follows the block.', + blockKey: tag.blockKey, + } + ) +} + +/** + * Reports each literal `` whose head names no block once the apply's renames are known + * (`renamed`, stored key → source key). The apply rewrites a renamed block's stored key to its new + * one, so such a head still resolves and stays `W_REF_NOT_NORMALIZED`; any other names no block, + * which the run cannot resolve: `W_UNRESOLVED_REF`, refused on a block the edit changed, as the + * server's lint refuses a dangling reference. + */ +export function checkUnresolvedTags( + ctx: CompileContext, + renamed: ReadonlyMap +): void { + for (const tag of ctx.unresolvedTags) { + if (renamed.has(tag.normalized)) { + reportNotNormalized(ctx, tag) + continue + } ctx.reporter.report( - 'W_REF_NOT_NORMALIZED', - normalized === head.name - ? `${token.value} names no block; the text is kept as written.` - : `${token.value} is not written with the block key ${normalized}; the text is kept as written.`, - rangeOf(ctx, node), + 'W_UNRESOLVED_REF', + `${tag.token} names no block, so it does not resolve at run time.`, + tag.at, { - hint: target?.ident - ? `Write \${${formatMemberPath(target.ident, segments.slice(1))}} in a template, so a rename follows the block.` - : 'Reference a declared block through its ident, so a rename follows the block.', - blockKey: site.decl.key, + hint: 'Reference a declared block through its ident, so a rename follows the block.', + blockKey: tag.blockKey, } ) } @@ -732,17 +770,20 @@ function segmentsOf( */ function checkEnvName(ctx: CompileContext, name: string, at: Range, blockKey: BlockKey): void { ctx.envReferences.add(name) - if (!ctx.envNames || ctx.envNames.has(name)) return - const near = didYouMean(name, ctx.envNames) + const envNames = ctx.envNames + if (!envNames || envNames.has(name)) return ctx.reporter.report( 'W_ENV_UNKNOWN', `The workspace has no env variable ${name}, so it resolves to nothing at run time.`, at, { - hint: ctx.envDeclaredIn - ? `${near ? `${near} Otherwise d` : 'D'}eclare ${name} in ${ctx.envDeclaredIn} and set its value with \`sim secrets set ${name} --scope workspace\`.` - : (near ?? - `Add ${name} under the workspace's or your environment variables, or use a name .sim/sdk/workspace.d.ts lists.`), + hint: () => { + const near = didYouMean(name, envNames) + return ctx.envDeclaredIn + ? `${near ? `${near} Otherwise d` : 'D'}eclare ${name} in ${ctx.envDeclaredIn} and set its value with \`sim secrets set ${name} --scope workspace\`.` + : (near ?? + `Add ${name} under the workspace's or your environment variables, or use a name .sim/sdk/workspace.d.ts lists.`) + }, blockKey, } ) @@ -997,7 +1038,7 @@ export function checkCodeFileRefs( hint: suggested ? `${ref.tag ? 'A tag' : REF_ROOT} names a block by its key: write ${rewritten(suggested.key)}.` : ctx.knownKeys.has(head.name) - ? formerBlockHint(ctx, head.name, renamed, rewritten, spell) + ? () => formerBlockHint(ctx, head.name, renamed, rewritten, spell) : () => didYouMean(head.name, ctx.declsByKey.keys(), (key) => spell([{ kind: 'name', name: key }]) diff --git a/packages/workflow-code/src/diagnostics/refusals.test.ts b/packages/workflow-code/src/diagnostics/refusals.test.ts index e5f0d2f95bb..6f24a04d8fb 100644 --- a/packages/workflow-code/src/diagnostics/refusals.test.ts +++ b/packages/workflow-code/src/diagnostics/refusals.test.ts @@ -83,6 +83,34 @@ describe('refuseEditedFindings', () => { expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) }) + it('refuses a literal tag naming no block on a changed block, as the server lint does', () => { + const compiled = compile([ + START, + "const call = api('Call', { url: 'https://x/' })", + "const cased = api('Cased', { url: 'https://x/' })", + "const bare = api('Bare', { url: 'https://x/' })", + 'start.to(call)', + ]) + + expect(errors(refuseEditedFindings(compiled, edited('call', 'cased', 'bare')))).toEqual([ + 'E_APPLY_UNRESOLVED_REF call', + ]) + expect(errors(refuseEditedFindings(compiled, edited('cased', 'bare')))).toEqual([]) + }) + + it('keeps a literal tag naming a block the source renamed, which the apply rewrites', () => { + const text = `// @sim-sdk 1\n${IMPORTS}\n\nexport default workflow('Test', () => {\n ${START}\n const call = api('Call', { url: 'https://x/' })\n})\n` + const compiled = compileWorkflow({ + entry: { path: 'workflow.ts', text }, + files: new Map(), + catalog: TEST_CATALOG, + sdkMajor: 1, + renamedKeys: new Map([['former', 'call']]), + }) + + expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) + }) + it('refuses an iteration field a shared code file reads for a block outside the loop', () => { const compiled = compile( [ diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index f6f3a127694..fe61b00b63a 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -105,9 +105,13 @@ const PARAM_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OPTION', 'E_APPLY_INVALID_SUBBLOCK_FIELD'], ]) -/** In `workflow.ts` on a block the edit changed: a path no output of the block has. */ +/** + * In `workflow.ts` on a block the edit changed: a path no output of the block has, or a literal + * `` whose head names no block, which the server's lint refuses as a dangling reference. + */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], + ['W_UNRESOLVED_REF', 'E_APPLY_UNRESOLVED_REF'], ]) /** From 5113eecf3f3daa7fcf06c0f99d26506e7d2e8f63 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 00:52:20 -0800 Subject: [PATCH 05/28] fix(workflow-code): warn on a new block of an uncatalogued type and refuse it as the placement gate does --- apps/docs/content/docs/cli/reference.mdx | 4 +- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- apps/docs/openapi-v2-workflows.json | 9 ++++ packages/sim-cli/src/generated/v2-api.ts | 9 ++++ packages/workflow-code/src/compile/blocks.ts | 44 ++++++++++++++----- .../workflow-code/src/diagnostics/codes.ts | 6 +++ .../src/diagnostics/refusals.test.ts | 32 ++++++++++++++ .../workflow-code/src/diagnostics/refusals.ts | 6 ++- 9 files changed, 99 insertions(+), 15 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index e4f0517f93b..4e17d50caa6 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -10018,7 +10018,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 76c3095768b..288570691eb 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index c91beb1ebea..317c16219dc 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -787,7 +787,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 98e30b39fa9..101dc9ca9f6 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -19204,6 +19204,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -19675,6 +19676,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -20664,6 +20666,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -21629,6 +21632,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -26683,6 +26687,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27259,6 +27264,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27678,6 +27684,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28543,6 +28550,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28962,6 +28970,7 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 47f56e72e70..ce952435617 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -874,6 +874,7 @@ export type ApplyWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1372,6 +1373,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1663,6 +1665,7 @@ export type ApplyWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -5334,6 +5337,7 @@ export type CreateWorkflowFromSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -10440,6 +10444,7 @@ export type GetWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -11132,6 +11137,7 @@ export type GetWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14459,6 +14465,7 @@ export type PlanWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14912,6 +14919,7 @@ export type PlanWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -15203,6 +15211,7 @@ export type PlanWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 0e486dbb5c2..ceec699ab4d 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1951,7 +1951,10 @@ function compileContainer(ctx: CompileContext, decl: BlockDecl): CompiledBlock | /** Pass 2 for one declaration. `null` for declarations already reported as invalid. */ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBlock | null { if (decl.duplicate || decl.builder.kind === 'unknown') return null - if (decl.builder.kind !== 'container') checkLegacyType(ctx, decl) + if (decl.builder.kind !== 'container') { + checkLegacyType(ctx, decl) + checkUncataloguedType(ctx, decl) + } return decl.builder.kind === 'container' ? compileContainer(ctx, decl) : compileRegularBlock(ctx, decl) @@ -1963,20 +1966,41 @@ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBloc * offline check, which has only the lock, stays quiet on a block the lock binds by key or ident, * as the server does for the stored block it continues. */ +/** Whether the lock of an unread stored workflow binds `decl`, by key or by ident. */ +function lockBinds(ctx: CompileContext, decl: BlockDecl): boolean { + const lock = ctx.storedUnread + return ( + lock !== undefined && + (Object.hasOwn(lock.blocks, decl.key) || + Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) + ) +} + +/** `W_UNKNOWN_BLOCK_TYPE` for a new `block()` of a type the catalog does not hold. */ +function checkUncataloguedType(ctx: CompileContext, decl: BlockDecl): void { + const { builder } = decl + if (builder.kind !== 'generic' || builder.block || lockBinds(ctx, decl)) return + const stored = Object.values(ctx.stored?.blocks ?? {}) + if (stored.some((block) => block.type === builder.type)) return + ctx.reporter.report( + 'W_UNKNOWN_BLOCK_TYPE', + `${builder.type} is not a block type this workspace offers, so an apply cannot add the block.`, + rangeOf(ctx, decl.call), + { + hint: () => + didYouMean(builder.type, Object.keys(ctx.catalog.blocks)) ?? + 'Use a block type from the catalog, or a builder such as api() or fn().', + blockKey: decl.key, + } + ) +} + function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { const block = catalogBlockOf(decl) const sunset = block?.lifecycle?.sunset if (!block || !sunset) return const stored = ctx.stored ? getOwn(ctx.stored.blocks, decl.key) : undefined - if (stored?.type === block.type) return - const lock = ctx.storedUnread - if ( - lock && - (Object.hasOwn(lock.blocks, decl.key) || - Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) - ) { - return - } + if (stored?.type === block.type || lockBinds(ctx, decl)) return const replacement = sunset.replacedBy const namespaces = replacement ? [ diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index de3caaa0826..c47104e4880 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -210,6 +210,12 @@ const COMPILE_CATALOG_CODES = [ 'W_UNKNOWN_MODEL', /** A block of a legacy or deprecated type; `hint` names the type that replaces it. */ 'W_LEGACY_BLOCK_TYPE', + /** + * A new `block()` of a type the catalog does not hold: the server refuses to place it + * (`E_APPLY_INVALID_BLOCK_TYPE`). A block the stored workflow or the lock already holds keeps + * its type quietly, so a workflow saved with a retired type still round-trips. + */ + 'W_UNKNOWN_BLOCK_TYPE', /** * A schedule created from source with no `timezone`: the editor seeds the viewer's zone, but * a schedule with none runs in UTC. diff --git a/packages/workflow-code/src/diagnostics/refusals.test.ts b/packages/workflow-code/src/diagnostics/refusals.test.ts index 6f24a04d8fb..84eaa5bbbe8 100644 --- a/packages/workflow-code/src/diagnostics/refusals.test.ts +++ b/packages/workflow-code/src/diagnostics/refusals.test.ts @@ -111,6 +111,38 @@ describe('refuseEditedFindings', () => { expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) }) + it('refuses a new block of a type the catalog does not hold, as the placement gate does', () => { + for (const type of ['slak', 'toString', '__proto__']) { + const text = `// @sim-sdk 1\nimport { block, trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Test', () => {\n ${START}\n const a = block('${type}', 'A', { text: 'hi' })\n})\n` + const compiled = compileWorkflow({ + entry: { path: 'workflow.ts', text }, + files: new Map(), + catalog: TEST_CATALOG, + sdkMajor: 1, + }) + expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([ + 'E_APPLY_INVALID_BLOCK_TYPE a', + ]) + } + }) + + it('keeps a block of an uncatalogued type the lock already binds', () => { + const text = `// @sim-sdk 1\nimport { block, trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Test', () => {\n ${START}\n const a = block('retired_kind', 'A', { text: 'hi' })\n})\n` + const compiled = compileWorkflow({ + entry: { path: 'workflow.ts', text }, + files: new Map(), + catalog: TEST_CATALOG, + sdkMajor: 1, + storedUnread: { + version: 1, + blocks: { a: { id: 'b-a', parent: null, ident: 'a', order: 0 } }, + variables: {}, + }, + }) + expect(compiled.diagnostics.map((d) => d.code)).toEqual([]) + expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([]) + }) + it('refuses an iteration field a shared code file reads for a block outside the loop', () => { const compiled = compile( [ diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index fe61b00b63a..1f5f429c145 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -66,6 +66,7 @@ export type EditRefusalCode = | 'E_APPLY_REF_OUT_OF_SCOPE' | 'E_APPLY_CODE_FILE_REF_SYNTAX' | 'E_APPLY_INVALID_SUBBLOCK_FIELD' + | 'E_APPLY_INVALID_BLOCK_TYPE' /** What an edit changed, as the caller can tell it. */ export interface EditedBlocks { @@ -107,11 +108,13 @@ const PARAM_REFUSALS: ReadonlyMap = new Map([ /** * In `workflow.ts` on a block the edit changed: a path no output of the block has, or a literal - * `` whose head names no block, which the server's lint refuses as a dangling reference. + * `` whose head names no block, which the server's lint refuses as a dangling reference, + * or a new block of a type the catalog does not hold, which its placement gate refuses. */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], ['W_UNRESOLVED_REF', 'E_APPLY_UNRESOLVED_REF'], + ['W_UNKNOWN_BLOCK_TYPE', 'E_APPLY_INVALID_BLOCK_TYPE'], ]) /** @@ -136,6 +139,7 @@ export const REFUSABLE_WARNING_CODES: ReadonlySet = new S 'W_UNKNOWN_OUTPUT_PATH', 'W_UNRESOLVED_REF', 'W_CODE_FILE_REF_SYNTAX', + 'W_UNKNOWN_BLOCK_TYPE', ]) /** The rules' error codes with the warning each refuses, for help text. */ From b5c42d9c1a423a6b67c642c4fd3a0c2932720d5e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 00:53:44 -0800 Subject: [PATCH 06/28] fix(workflow-code): keep ids by key when two block consts swap and both keys survive --- .../workflow-code/src/materialize/identity.ts | 8 +++++--- .../workflow-code/src/materialize/index.test.ts | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 853cbe696fd..5c89586a2b6 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -203,8 +203,10 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { const open = () => keys.filter((key) => !resolved.has(key) && !settled.has(key)) /** - * A key whose ident in the file now belongs to another lock entry: in a chain rename (A takes - * B's name while B takes a new one) the key is B's, but the block is A's, so rule 2 binds it. + * A key whose ident in the file now belongs to another lock entry whose key the file dropped: + * in a chain rename (A takes B's name while B takes a new one) the key is B's, but the block is + * A's, so rule 2 binds it. When the owner's key survives (two consts swapped), the key keeps its + * own entry, as rule 2 and `lockIdentRenames` read only a dropped key. */ const identOwner = new Map() for (const [key, entry] of Object.entries(lock?.blocks ?? {})) { @@ -215,7 +217,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { const entry = lockEntry(key) if (ident === undefined || !entry || entry.ident === ident) return false const owner = identOwner.get(ident) - return owner !== undefined && owner !== key + return owner !== undefined && owner !== key && !Object.hasOwn(workflow.blocks, owner) } /** Stale lock entries whose id left prev; reported only if no later rule matches the key. */ diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 519285156ef..723778e1bb9 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -592,6 +592,22 @@ describe('materialize: identity', () => { expect(result.entries.filter((entry) => entry.status !== 'matched')).toEqual([]) }) + it('keeps each id by key when two blocks swap their consts and both keys survive', () => { + const prev = base() + const lock = lockFor(prev) + const result = run(canonical(prev, lock).workflow, lock, { + prev, + compiledIdents: { + ...Object.fromEntries(Object.entries(lock.blocks).map(([key, b]) => [key, b.ident])), + fetch: lock.blocks.report.ident, + report: lock.blocks.fetch.ident, + }, + }) + expect(result.identity.blocks.fetch.id).toBe(lock.blocks.fetch.id) + expect(result.identity.blocks.report.id).toBe(lock.blocks.report.id) + expect(result.entries.filter((entry) => entry.status !== 'matched')).toEqual([]) + }) + it('infers a lockless rename from type, parent and agreeing neighbours, flagged', () => { const prev = base() const result = run(renameFetch(canonical(prev).workflow), null, { prev }) @@ -1309,7 +1325,6 @@ describe('materialize: preserved UI state comes from prev', () => { it.each([ ['a chain', { fetch: 'report', report: 'summary' }, 'saw and '], - ['a swap', { fetch: 'report', report: 'fetch' }, 'saw and '], ])( 'rewrites a restored inactive value through %s of renames at once', (_label, moves, expected) => { From 05c590bde7dc355ec21cddfab84113ef626c010f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 00:54:21 -0800 Subject: [PATCH 07/28] fix(workflow-code): keep the stored variable order when an apply changes a variable --- .../src/materialize/index.test.ts | 22 +++++++++++++++++ .../workflow-code/src/materialize/index.ts | 24 +++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 723778e1bb9..8b624ff52dc 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -974,6 +974,28 @@ describe('materialize: identity', () => { expect(ids(renamed('count', 'number', 'Rows to fetch'))).toEqual(['new-1', 'f-q']) }) + it('keeps the stored variable order when source lists the variables in another order', () => { + const pulled = workflowState([blockState('a1', 'function', 'Fetch', { code: 'return 1' })], { + variables: { + 'var-z': { id: 'var-z', name: 'Zeta', type: 'number', value: 1 }, + 'var-a': { id: 'var-a', name: 'Alpha', type: 'number', value: 1 }, + }, + }) + const edited = edit(canonical(pulled).workflow, (draft) => { + draft.variables = { + alpha: { ...draft.variables.alpha, value: 2 }, + new: { key: 'new', name: 'New', type: 'number', value: 3 }, + zeta: draft.variables.zeta, + } + }) + + expect(Object.keys(run(edited, null, { prev: pulled }).state.variables ?? {})).toEqual([ + 'var-z', + 'var-a', + 'new-1', + ]) + }) + it('keeps the id of the one variable renamed to a key of the same type', () => { const pulled = base() const renamed = (type: 'number' | 'plain') => diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index f353768b423..a8afd0ba639 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -40,7 +40,7 @@ import { rewriteRenamedReferences, rewriteStoredBlockReferences } from './values * Variables keep their ids by key: the lock's id, then prev's, else a new one. When exactly one * variable's key is new and exactly one prev variable's key is gone, with the same type, the * variable is a rename and keeps that id. A variable canonical-equal to the prev variable with - * that id is prev's stored object unchanged. + * that id is prev's stored object unchanged, and continuing variables keep their stored order. */ function writeVariables( workflow: CanonicalWorkflow, @@ -103,7 +103,27 @@ function writeVariables( : { ...stored, id, name: variable.name, type: variable.type, value: variable.value } ) } - return { variables, idByKey } + return { variables: inStoredOrder(variables, prevVariables), idByKey } +} + +/** + * `variables` with the ones `stored` holds first, in its order, then the new ones in source order. + * The panel lists variables in record order and the printer sorts them by key, so writing them in + * source order would reorder the panel on every apply that touches a variable. + */ +function inStoredOrder( + variables: Record, + stored: Readonly> +): Record { + const ordered: Record = {} + for (const id of Object.keys(stored)) { + const variable = getOwn(variables, id) + if (variable) setOwn(ordered, id, variable) + } + for (const [id, variable] of Object.entries(variables)) { + if (!Object.hasOwn(ordered, id)) setOwn(ordered, id, variable) + } + return ordered } /** The one variable a lone key rename left without a known id, with its id (see {@link soleRename}). */ From a9ed7dd1fd32ab32bf6f31dd9582f7d93400517d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 00:57:17 -0800 Subject: [PATCH 08/28] fix(workspace-source): stop reading managed table rows once they pass the rows byte limit --- .../application/kinds/table/adapter.ts | 6 +++-- .../application/kinds/table/rows.ts | 15 +++++++++-- .../application/rows.integration.ts | 26 +++++++++++++++++++ .../workspace-code/src/kinds/table-rows.ts | 6 +++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 1b07e4359c8..598778c5679 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -409,7 +409,7 @@ async function planRows( return outcome } const rowKey = rows.key ?? '' - const stored = await readStoredRows(ctx, state.table, keyById) + const stored = await readStoredRows(ctx, state.table, keyById, rowKey) const kept = unkeyedRows(stored, rowKey) if (kept.length > 0) { outcome.diagnostics.push(unkeyedFinding(key, state.table.name, rowKey, kept.length)) @@ -774,7 +774,9 @@ async function rowsToPrint( ) { const { decl: storedDecl, keyById } = tableAsDecl(state, bindings) const rowKey = storedDecl.rows?.key - if (rowKey !== undefined) return { ...(await readStoredRows(ctx, state.table, keyById)), rowKey } + if (rowKey !== undefined) { + return { ...(await readStoredRows(ctx, state.table, keyById, rowKey)), rowKey } + } return decl?.rows?.mode === 'seed' ? { seed: decl.rows } : undefined } diff --git a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts index 42daa42e64f..3d88c6877ae 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts @@ -8,6 +8,7 @@ import type { Json } from '@sim/workflow-code' import type { BaseRows } from '@sim/workspace-code' import { hasRowKey, + printedRowBytes, type TableRowCells, type TableRowsDecl, tableRowsFingerprint, @@ -55,12 +56,14 @@ function sourceCell(column: ColumnDefinition, value: unknown): Json { /** * Reads every row of `table`: the cells of each column `keyById` names (column id → column key). * The version is read first, so rows written while the listing pages read as a server change on - * the next apply. + * the next apply. The rows a rows file prints (those holding `rowKey`) are counted as they page + * in, and the read stops once they pass `maxRowsBytes`, which no directory could hold. */ export async function readStoredRows( ctx: SourceContext, table: TableDefinition, - keyById: ReadonlyMap + keyById: ReadonlyMap, + rowKey: string ): Promise { if (table.rowCount > WORKSPACE_SOURCE_LIMITS.maxTableRows) { throw new OrchestrationError( @@ -72,6 +75,7 @@ export async function readStoredRows( const columnsById = new Map(table.schema.columns.map((column) => [getColumnId(column), column])) const data: TableRowCells[] = [] const byColumnId: RowData[] = [] + let printedBytes = 0 let cursor: string | undefined do { const page = await listTableRows.execute({ @@ -92,6 +96,13 @@ export async function readStoredRows( } data.push(cells) byColumnId.push(row.data) + if (hasRowKey(cells, rowKey)) printedBytes += printedRowBytes(cells) + } + if (printedBytes > WORKSPACE_SOURCE_LIMITS.maxRowsBytes) { + throw new OrchestrationError( + 'payload_too_large', + `The rows of table "${table.name}" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte limit a directory's rows files hold.` + ) } cursor = page.nextCursor ?? undefined } while (cursor) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 00b465709fc..9eb6f19089c 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -812,6 +812,32 @@ export default defineTable({ } ) + check( + 'stops reading managed rows once they pass the rows byte limit a directory holds', + async () => { + const workspaceId = await newWorkspace() + await managedPlans(workspaceId) + const filler = 'x'.repeat(370_000) + const rows = Math.ceil(WORKSPACE_SOURCE_LIMITS.maxRowsBytes / filler.length) + 1 + for (let index = 0; index < rows; index++) { + await addUiRow(workspaceId, { code: `big${index}-${filler}`, seats: index }) + } + + const read = await runner.get(workspaceId) + + expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() + expect(read.skipped).toEqual([ + { + kind: 'table', + key: 'plans', + reason: expect.stringContaining( + `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` + ), + }, + ]) + } + ) + check('seeds a new table once, and only a new or empty one', async () => { const workspaceId = await newWorkspace() const files = { 'tables/plans.ts': SEED, 'tables/plans.rows.jsonl': ROWS } diff --git a/packages/workspace-code/src/kinds/table-rows.ts b/packages/workspace-code/src/kinds/table-rows.ts index 2aa21400c6b..fecda4e0e58 100644 --- a/packages/workspace-code/src/kinds/table-rows.ts +++ b/packages/workspace-code/src/kinds/table-rows.ts @@ -22,6 +22,7 @@ import { isRecordLike } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import type { Json } from '@sim/workflow-code' +import { utf8Bytes } from '@sim/workflow-code/limits' import type { ResourceFingerprint } from '../base' import { diagnostic, @@ -73,6 +74,11 @@ export function hasRowKey(row: TableRowCells, rowKey: string): boolean { return rowKeyText(row[rowKey]) !== undefined } +/** The UTF-8 bytes `row` takes as one line of a rows file, its newline included. */ +export function printedRowBytes(row: TableRowCells): number { + return utf8Bytes(JSON.stringify(filledCells(row))) + 1 +} + /** A row without its empty cells (null, `''` and `[]`, which a table stores as no value). */ function filledCells(row: TableRowCells): TableRowCells { const cells: TableRowCells = {} From 08e3295c5a7a5d96540dd1c9df1c39aac08fca9c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:01:30 -0800 Subject: [PATCH 09/28] fix(workspace-source): read only the deployments a plan declares or its base records --- .../application/deployments.integration.ts | 33 +++++- .../application/kinds/deployment/adapter.ts | 101 +++++++++++++----- 2 files changed, 105 insertions(+), 29 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/deployments.integration.ts b/apps/sim/lib/workspace-source/application/deployments.integration.ts index 6f645c8a56c..1091ccce697 100644 --- a/apps/sim/lib/workspace-source/application/deployments.integration.ts +++ b/apps/sim/lib/workspace-source/application/deployments.integration.ts @@ -32,7 +32,10 @@ import { replaceWorkflowChatDeployment } from '@/lib/chat-deployments/applicatio import { OrchestrationError } from '@/lib/core/orchestration/types' import { encryptSecret } from '@/lib/core/security/encryption' import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' -import { deployWorkflow } from '@/lib/workflows/application/deployments' +import { + deployWorkflow, + readWorkflowDeploymentStatus, +} from '@/lib/workflows/application/deployments' import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' import { listWorkflowTriggers } from '@/lib/workflows/application/triggers' import { updateWorkflowPublicApi } from '@/lib/workflows/application/update-workflow-deployment-settings' @@ -1181,6 +1184,34 @@ describe('workspace source: deployments', () => { } ) + check( + 'reads the deployment state of only the workflows the directory declares or its base records', + async () => { + const { workspaceId, id, files, base } = await deployedWorkspace() + const others: Files = {} + for (const name of ['alpha', 'beta', 'gamma']) { + others[`workflows/${name}/workflow.ts`] = NIGHTLY.replace( + "workflow('Nightly'", + `workflow('${name}'` + ) + others[`deployments/${name}.ts`] = DRAFT + } + const deployedOthers = await apply(workspaceId, others) + expect(deployedOthers.failed).toBeUndefined() + + const reads = vi.spyOn(readWorkflowDeploymentStatus, 'execute') + try { + const plan = await apply(workspaceId, files, { base, kinds: ['deployment'], dryRun: true }) + expect(plan.failed).toBeUndefined() + expect(new Set(reads.mock.calls.map(([call]) => call.input.workflowId))).toEqual( + new Set([id]) + ) + } finally { + reads.mockRestore() + } + } + ) + check('refuses a deployment of a workflow the workspace does not hold', async () => { const workspaceId = await workspaces.create({ name: 'Deploy nothing', ownerId }) const refused = await refusal(apply(workspaceId, { 'deployments/missing.ts': DRAFT })) diff --git a/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts index 523d3877984..4ccf679ecc4 100644 --- a/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts @@ -89,6 +89,7 @@ import type { PlanContext, PlannedChange, SourceContext, + StoredResource, StoredSet, } from '@/lib/workspace-source/application/types' import { resolveIds } from '@/lib/workspaces/resource-keys' @@ -96,9 +97,15 @@ import { resolveIds } from '@/lib/workspaces/resource-keys' /** Deployed workflows per page while listing them. */ const LIST_PAGE = 200 -/** Deployments a load reads at once. */ +/** Deployments a plan or a get reads at once. */ const READ_CONCURRENCY = 4 +/** + * A listed deployed workflow's settings: `null` until read. A load lists deployed workflows only; + * a plan reads the settings of those it needs ({@link readNeeded}) and a get those it prints. + */ +type ListedDeployment = StoredDeployment | null + /** The base entry field naming the variable a chat password was last set from. */ const PASSWORD_ENV = 'x-passwordEnv' @@ -491,7 +498,7 @@ async function planLiveDraft( async function planDeployment( key: string, decl: DeploymentDecl, - stored: StoredSet, + stored: StoredSet, ctx: PlanContext ): Promise { const change: DeploymentChange = { @@ -507,7 +514,8 @@ async function planDeployment( planned.kind === 'workflow' && planned.key === key ) const creating = workflowChange?.action === 'create' - const deployed = stored.bound.get(key) + const listed = stored.bound.get(key) + const deployed = listed?.state ? { ...listed, state: listed.state } : undefined const workflowId = deployed?.id ?? workflowChange?.storedId ?? @@ -837,23 +845,22 @@ function listAllDeployedWorkflows(ctx: SourceContext) { }) } -export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { +export const deploymentAdapter: KindAdapter<'deployment', ListedDeployment> = { codec: deploymentCodec, async load(ctx, { allocate }) { const workflows = await listDeployedWorkflows(ctx) - const records = await mapWithConcurrency(workflows, READ_CONCURRENCY, async (workflow) => { - const state = await readDeployment(ctx, workflow.id) - return state ? [{ id: workflow.id, name: workflow.name, state }] : [] - }) - return bindStored(ctx, 'workflow', records.flat(), { allocate }) + const records = workflows.map(({ id, name }) => ({ id, name, state: null })) + return bindStored(ctx, 'workflow', records, { allocate }) }, - async plan(desired, stored, ctx) { + async plan(desired, listed, ctx) { + const stored = await readNeeded(listed, desired, ctx) const changes: DeploymentChange[] = [] for (const [key, decl] of desired) changes.push(await planDeployment(key, decl, stored, ctx)) const undeclared = planUndeclared('deployment', desired, stored, ctx, { - fingerprint: (record, base) => storedFingerprint(record.state, passwordEnvOf(base)), + fingerprint: (record, base) => + record.state ? storedFingerprint(record.state, passwordEnvOf(base)) : '', detail: (record, base) => ({ workflowId: record.id, deploy: false, @@ -866,7 +873,7 @@ export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { const refused = change.diagnostics.find( (finding) => finding.code === 'E_APPLY_DELETE_NOT_ALLOWED' ) - if (!record || (change.action !== 'delete' && !refused)) continue + if (!record?.state || (change.action !== 'delete' && !refused)) continue const counts = await stopsOnUndeploy(ctx, record.id, record.state) if (change.destructive) change.destructive = { what: 'deployment', counts } if (refused) { @@ -1005,26 +1012,64 @@ export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { } }, - async read(stored) { - const { state } = stored - // A draft that differs reads as pinned, so applying what a get wrote never publishes it. - const decl = declOf(state, { live: state.needsRedeployment ? 'pinned' : 'draft' }) - if (!decl) { - throw new OrchestrationError( - 'validation', - state.chat?.authType === null - ? 'Its chat gates access in a way source cannot name (public, password, emails or SSO).' - : 'Its chat answers with a block the draft no longer has, which source cannot name.' + read: readListed, + + async readMany(stored, ctx) { + const reads = await mapWithConcurrency(stored, READ_CONCURRENCY, async (resource) => { + const outcome = await readListed(resource, ctx).then( + (read) => ({ read }), + (error: unknown) => ({ error }) ) - } - return { - text: deploymentCodec.print(stored.key, decl).text, - blobs: [], - base: baseEntry(stored.id, state, undefined), - } + return [resource.key, outcome] as const + }) + return new Map(reads) }, } +/** + * The listed deployments a plan reads the settings of: those the directory declares and those the + * base records, which a plan may update, adopt or undeploy. The rest stay unread (`null`): the plan + * only says they are kept. A workflow undeployed since the listing drops out, as if never listed. + */ +async function readNeeded( + listed: StoredSet, + desired: ReadonlyMap, + ctx: PlanContext +): Promise> { + const bound = await mapWithConcurrency( + [...listed.bound], + READ_CONCURRENCY, + async ([key, record]) => { + const recorded = ctx.base?.resources[resourceAddress('deployment', key)]?.id === record.id + if (!desired.has(key) && !recorded) return [[key, record] as const] + const state = await readDeployment(ctx, record.id) + return state ? [[key, { ...record, state }] as const] : [] + } + ) + return { bound: new Map(bound.flat()), unbound: listed.unbound } +} + +/** A listed deployment as source, its settings read now. */ +async function readListed(stored: StoredResource, ctx: SourceContext) { + const state = await readDeployment(ctx, stored.id) + if (!state) throw new OrchestrationError('not_found', 'The workflow is no longer deployed.') + // A draft that differs reads as pinned, so applying what a get wrote never publishes it. + const decl = declOf(state, { live: state.needsRedeployment ? 'pinned' : 'draft' }) + if (!decl) { + throw new OrchestrationError( + 'validation', + state.chat?.authType === null + ? 'Its chat gates access in a way source cannot name (public, password, emails or SSO).' + : 'Its chat answers with a block the draft no longer has, which source cannot name.' + ) + } + return { + text: deploymentCodec.print(stored.key, decl).text, + blobs: [], + base: baseEntry(stored.id, state, undefined), + } +} + /** * Warns on every workflow this apply edits that is deployed while no deployment file in the * directory decides what happens to its live version (`W_DEPLOYED_DRAFT_CHANGED`): the edit From 81662589a0ad7d7c3152968c9b27882d96c6c82a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:04:07 -0800 Subject: [PATCH 10/28] refactor(workflow-code): tidy the qa6 fixes (shared list helper, lazy hints, test helpers) --- packages/workflow-code/src/compile/blocks.ts | 12 ++-- packages/workflow-code/src/compile/context.ts | 1 + packages/workflow-code/src/compile/values.ts | 18 +++-- .../src/diagnostics/refusals.test.ts | 56 +++++++--------- .../workflow-code/src/materialize/identity.ts | 32 ++++----- .../src/materialize/index.test.ts | 67 +++++++++---------- 6 files changed, 89 insertions(+), 97 deletions(-) diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index ceec699ab4d..764f191a869 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1960,12 +1960,6 @@ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBloc : compileRegularBlock(ctx, decl) } -/** - * `W_LEGACY_BLOCK_TYPE` for a new block of a legacy or deprecated type, naming its replacement and - * that type's builders. A block the stored workflow already holds under that type stays quiet; an - * offline check, which has only the lock, stays quiet on a block the lock binds by key or ident, - * as the server does for the stored block it continues. - */ /** Whether the lock of an unread stored workflow binds `decl`, by key or by ident. */ function lockBinds(ctx: CompileContext, decl: BlockDecl): boolean { const lock = ctx.storedUnread @@ -1995,6 +1989,12 @@ function checkUncataloguedType(ctx: CompileContext, decl: BlockDecl): void { ) } +/** + * `W_LEGACY_BLOCK_TYPE` for a new block of a legacy or deprecated type, naming its replacement and + * that type's builders. A block the stored workflow already holds under that type stays quiet; an + * offline check, which has only the lock, stays quiet on a block the lock binds by key or ident, + * as the server does for the stored block it continues. + */ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { const block = catalogBlockOf(decl) const sunset = block?.lifecycle?.sunset diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index 54ad9191a93..21bd885d28d 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -199,6 +199,7 @@ export interface CompileContext { /** A `` in literal text whose head is no declared block's key. */ export interface LiteralTag { token: string + segments: RefSegment[] normalized: BlockKey at: Range blockKey: BlockKey diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index 67976378dd3..28e197f9846 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -270,7 +270,13 @@ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: V checkIterationTag(ctx, rangeOf(ctx, node), site.decl.key, segments) continue } - const tag = { token: token.value, normalized, at: rangeOf(ctx, node), blockKey: site.decl.key } + const tag = { + token: token.value, + segments, + normalized, + at: rangeOf(ctx, node), + blockKey: site.decl.key, + } if (!ctx.declsByKey.has(normalized) && segments.length > 1) ctx.unresolvedTags.push(tag) else reportNotNormalized(ctx, tag) } @@ -278,7 +284,6 @@ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: V function reportNotNormalized(ctx: CompileContext, tag: LiteralTag): void { const target = ctx.declsByKey.get(tag.normalized) - const segments = parseStoredPath(tag.token.slice(1, -1)) ?? [] ctx.reporter.report( 'W_REF_NOT_NORMALIZED', target @@ -286,9 +291,10 @@ function reportNotNormalized(ctx: CompileContext, tag: LiteralTag): void { : `${tag.token} names no block; the text is kept as written.`, tag.at, { - hint: target?.ident - ? `Write \${${formatMemberPath(target.ident, segments.slice(1))}} in a template, so a rename follows the block.` - : 'Reference a declared block through its ident, so a rename follows the block.', + hint: () => + target?.ident + ? `Write \${${formatMemberPath(target.ident, tag.segments.slice(1))}} in a template, so a rename follows the block.` + : 'Reference a declared block through its ident, so a rename follows the block.', blockKey: tag.blockKey, } ) @@ -649,7 +655,7 @@ function compileRef(ctx: CompileContext, node: Node, site: ValueSite): string | `No workflow variable is keyed ${name}.`, rangeOf(ctx, first), { - hint: didYouMean(key, ctx.variables.keys(), (known) => `vars.${known}`), + hint: () => didYouMean(key, ctx.variables.keys(), (known) => `vars.${known}`), blockKey: site.decl.key, } ) diff --git a/packages/workflow-code/src/diagnostics/refusals.test.ts b/packages/workflow-code/src/diagnostics/refusals.test.ts index 84eaa5bbbe8..b0f1d8805ed 100644 --- a/packages/workflow-code/src/diagnostics/refusals.test.ts +++ b/packages/workflow-code/src/diagnostics/refusals.test.ts @@ -5,15 +5,21 @@ import { FINDING_LIMITS } from './bound' import { MAX_ERRORS } from './codes' import { type EditedBlocks, refuseEditedFindings } from './refusals' -const IMPORTS = "import { api, codeFile, fn, loop, t, trigger, workflow } from '@sim/workflow-sdk'" - -function compile(body: string[], files: Record = {}) { +const IMPORTS = + "import { api, block, codeFile, fn, loop, t, trigger, workflow } from '@sim/workflow-sdk'" + +function compile( + body: string[], + files: Record = {}, + options: Partial[0]> = {} +) { const text = `// @sim-sdk 1\n${IMPORTS}\n\nexport default workflow('Test', () => {\n${body.map((line) => ` ${line}`).join('\n')}\n})\n` return compileWorkflow({ entry: { path: 'workflow.ts', text }, files: new Map(Object.entries(files)), catalog: TEST_CATALOG, sdkMajor: 1, + ...options, }) } @@ -99,27 +105,18 @@ describe('refuseEditedFindings', () => { }) it('keeps a literal tag naming a block the source renamed, which the apply rewrites', () => { - const text = `// @sim-sdk 1\n${IMPORTS}\n\nexport default workflow('Test', () => {\n ${START}\n const call = api('Call', { url: 'https://x/' })\n})\n` - const compiled = compileWorkflow({ - entry: { path: 'workflow.ts', text }, - files: new Map(), - catalog: TEST_CATALOG, - sdkMajor: 1, - renamedKeys: new Map([['former', 'call']]), - }) + const compiled = compile( + [START, "const call = api('Call', { url: 'https://x/' })"], + {}, + { renamedKeys: new Map([['former', 'call']]) } + ) expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) }) it('refuses a new block of a type the catalog does not hold, as the placement gate does', () => { for (const type of ['slak', 'toString', '__proto__']) { - const text = `// @sim-sdk 1\nimport { block, trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Test', () => {\n ${START}\n const a = block('${type}', 'A', { text: 'hi' })\n})\n` - const compiled = compileWorkflow({ - entry: { path: 'workflow.ts', text }, - files: new Map(), - catalog: TEST_CATALOG, - sdkMajor: 1, - }) + const compiled = compile([START, `const a = block('${type}', 'A', { text: 'hi' })`]) expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([ 'E_APPLY_INVALID_BLOCK_TYPE a', ]) @@ -127,18 +124,17 @@ describe('refuseEditedFindings', () => { }) it('keeps a block of an uncatalogued type the lock already binds', () => { - const text = `// @sim-sdk 1\nimport { block, trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Test', () => {\n ${START}\n const a = block('retired_kind', 'A', { text: 'hi' })\n})\n` - const compiled = compileWorkflow({ - entry: { path: 'workflow.ts', text }, - files: new Map(), - catalog: TEST_CATALOG, - sdkMajor: 1, - storedUnread: { - version: 1, - blocks: { a: { id: 'b-a', parent: null, ident: 'a', order: 0 } }, - variables: {}, - }, - }) + const compiled = compile( + [START, "const a = block('retired_kind', 'A', { text: 'hi' })"], + {}, + { + storedUnread: { + version: 1, + blocks: { a: { id: 'b-a', parent: null, ident: 'a', order: 0 } }, + variables: {}, + }, + } + ) expect(compiled.diagnostics.map((d) => d.code)).toEqual([]) expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([]) }) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 5c89586a2b6..32a234b698f 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -361,17 +361,19 @@ interface InferenceState { type CanonicalEdges = CanonicalWorkflow['edges'] +/** Appends `item` to the list `map` holds under `key`, starting the list when there is none. */ +function pushTo(map: Map, key: K, item: V): void { + const list = map.get(key) + if (list) list.push(item) + else map.set(key, [item]) +} + /** Each block's edges, in or out, so a block's neighbourhood reads only its own edges. */ function edgesByEndpoint(edges: CanonicalEdges): Map { const byKey = new Map() - const add = (key: BlockKey, edge: CanonicalEdges[number]) => { - const list = byKey.get(key) - if (list) list.push(edge) - else byKey.set(key, [edge]) - } for (const edge of edges) { - add(edge.from, edge) - if (edge.to !== edge.from) add(edge.to, edge) + pushTo(byKey, edge.from, edge) + if (edge.to !== edge.from) pushTo(byKey, edge.to, edge) } return byKey } @@ -479,10 +481,7 @@ function settingsIndex(blocks: ReadonlyArray<[BlockKey, CanonicalBlock]>) { holdsAngle(value) ? `<\u0000${id}` : `=\u0000${id}\u0000${canonicalJson(value)}` for (const [key, block] of blocks) { for (const [id, value] of Object.entries(block.params)) { - const bucket = bucketOf(id, value) - const keys = byValue.get(bucket) - if (keys) keys.push(key) - else byValue.set(bucket, [key]) + pushTo(byValue, bucketOf(id, value), key) const held = holders.get(id) if (held) held.add(key) else holders.set(id, new Set([key])) @@ -520,11 +519,7 @@ function neighbourPrefix(items: Set, order: (left: string, right: string function invert(entries: Iterable<[BlockKey, Iterable]>): Map { const byItem = new Map() for (const [key, items] of entries) { - for (const item of items) { - const keys = byItem.get(item) - if (keys) keys.push(key) - else byItem.set(item, [key]) - } + for (const item of items) pushTo(byItem, item, key) } return byItem } @@ -658,10 +653,7 @@ function inferRenames(state: InferenceState): void { const referrersOf = new Map() for (const [other, block] of Object.entries(prevCanonical.blocks)) { for (const key of referencedKeys(block)) { - if (key === other || !Object.hasOwn(prevCanonical.blocks, key)) continue - const referrers = referrersOf.get(key) - if (referrers) referrers.push(other) - else referrersOf.set(key, [other]) + if (key !== other && Object.hasOwn(prevCanonical.blocks, key)) pushTo(referrersOf, key, other) } } const targetReferences = new Map>() diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 8b624ff52dc..a7c791ec7e1 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -1345,41 +1345,38 @@ describe('materialize: preserved UI state comes from prev', () => { expect(subBlockValue(result.state, 's1', 'text')).toBe('saw ') }) - it.each([ - ['a chain', { fetch: 'report', report: 'summary' }, 'saw and '], - ])( - 'rewrites a restored inactive value through %s of renames at once', - (_label, moves, expected) => { - const state = workflowState([ - blockState('a1', 'function', 'Fetch', { code: 'return 1' }), - blockState('a2', 'function', 'Report', { code: 'return 2' }), - blockState('s1', 'slack', 'Post', { - operation: 'read', - channel: 'C1', - text: 'saw and ', - }), - ]) - const lock = lockFor(state) - const renamed: Record = moves - const target = edit(canonical(state, lock).workflow, (draft) => { - const blocks = { ...draft.blocks } - draft.blocks = omit(draft.blocks, ['fetch', 'report']) - for (const [from, to] of Object.entries(renamed)) { - draft.blocks[to] = { ...blocks[from], key: to, name: to[0].toUpperCase() + to.slice(1) } - } - draft.blocks.post.params = omit(draft.blocks.post.params, ['text']) - }) - const result = run(target, lock, { - prev: state, - compiledIdents: Object.fromEntries( - Object.entries(renamed).map(([from, to]) => [to, lock.blocks[from].ident]) - ), - }) - expect(result.identity.blocks[renamed.fetch].id).toBe('a1') - expect(result.identity.blocks[renamed.report].id).toBe('a2') - expect(subBlockValue(result.state, 's1', 'text')).toBe(expected) - } - ) + it('rewrites a restored inactive value through a chain of renames at once', () => { + const state = workflowState([ + blockState('a1', 'function', 'Fetch', { code: 'return 1' }), + blockState('a2', 'function', 'Report', { code: 'return 2' }), + blockState('s1', 'slack', 'Post', { + operation: 'read', + channel: 'C1', + text: 'saw and ', + }), + ]) + const lock = lockFor(state) + const renamed: Record = { fetch: 'report', report: 'summary' } + const target = edit(canonical(state, lock).workflow, (draft) => { + const blocks = { ...draft.blocks } + draft.blocks = omit(draft.blocks, ['fetch', 'report']) + for (const [from, to] of Object.entries(renamed)) { + draft.blocks[to] = { ...blocks[from], key: to, name: to[0].toUpperCase() + to.slice(1) } + } + draft.blocks.post.params = omit(draft.blocks.post.params, ['text']) + }) + const result = run(target, lock, { + prev: state, + compiledIdents: Object.fromEntries( + Object.entries(renamed).map(([from, to]) => [to, lock.blocks[from].ident]) + ), + }) + expect(result.identity.blocks[renamed.fetch].id).toBe('a1') + expect(result.identity.blocks[renamed.report].id).toBe('a2') + expect(subBlockValue(result.state, 's1', 'text')).toBe( + 'saw and ' + ) + }) it('clears a standalone advanced field that source emptied instead of restoring it', () => { const state = workflowState([ From cbfbe4b4aaf4e080b403f731e397259cf4e3611f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:07:15 -0800 Subject: [PATCH 11/28] test(workflow-code): sharpen the qa6 test names and the lock cap fixture --- .../application/rows.integration.ts | 43 +++++++++---------- .../src/compile/ergonomics.test.ts | 4 +- packages/workflow-code/src/lock.test.ts | 5 ++- .../workflow-code/src/materialize/identity.ts | 1 - 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 9eb6f19089c..9f75505c97c 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -812,31 +812,28 @@ export default defineTable({ } ) - check( - 'stops reading managed rows once they pass the rows byte limit a directory holds', - async () => { - const workspaceId = await newWorkspace() - await managedPlans(workspaceId) - const filler = 'x'.repeat(370_000) - const rows = Math.ceil(WORKSPACE_SOURCE_LIMITS.maxRowsBytes / filler.length) + 1 - for (let index = 0; index < rows; index++) { - await addUiRow(workspaceId, { code: `big${index}-${filler}`, seats: index }) - } + check('skips a table whose managed rows pass the rows byte limit, naming the table', async () => { + const workspaceId = await newWorkspace() + await managedPlans(workspaceId) + const filler = 'x'.repeat(370_000) + const rows = Math.ceil(WORKSPACE_SOURCE_LIMITS.maxRowsBytes / filler.length) + 1 + for (let index = 0; index < rows; index++) { + await addUiRow(workspaceId, { code: `big${index}-${filler}`, seats: index }) + } - const read = await runner.get(workspaceId) + const read = await runner.get(workspaceId) - expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() - expect(read.skipped).toEqual([ - { - kind: 'table', - key: 'plans', - reason: expect.stringContaining( - `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` - ), - }, - ]) - } - ) + expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() + expect(read.skipped).toEqual([ + { + kind: 'table', + key: 'plans', + reason: expect.stringContaining( + `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` + ), + }, + ]) + }) check('seeds a new table once, and only a new or empty one', async () => { const workspaceId = await newWorkspace() diff --git a/packages/workflow-code/src/compile/ergonomics.test.ts b/packages/workflow-code/src/compile/ergonomics.test.ts index 61efa0c2b41..cf89e3c1ba8 100644 --- a/packages/workflow-code/src/compile/ergonomics.test.ts +++ b/packages/workflow-code/src/compile/ergonomics.test.ts @@ -489,8 +489,8 @@ describe('header, names and hand-written references', () => { ) expect(found).toEqual(warning ? [warning] : []) if (literal === "''") { - const warning = result.diagnostics.find((d) => d.code === 'W_REF_NOT_NORMALIZED') - expect(warning?.hint).toContain('${a.data}') + const diagnostic = result.diagnostics.find((d) => d.code === 'W_REF_NOT_NORMALIZED') + expect(diagnostic?.hint).toContain('${a.data}') } }) }) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index 19eb6e2b5c7..148a2bb799a 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -211,9 +211,10 @@ describe('parseWorkflowLock', () => { const over = SOURCE_LIMITS.maxBlocks + 1 const blocks: Record = {} const variables: Record = {} + // Each entry is malformed, so a reader that walked them would report far more than the cap. for (let i = 0; i < over; i += 1) { - blocks[`b${i}`] = { id: `id${i}`, parent: null, ident: `b${i}`, order: i } - variables[`v${i}`] = `var${i}` + blocks[`b${i}`] = { id: i } + variables[`v${i}`] = '' } expect(codesFor({ version: 1, blocks, variables: {} })).toEqual(['E_LOCK_SHAPE /blocks']) expect(codesFor({ version: 1, blocks: {}, variables })).toEqual(['E_LOCK_SHAPE /variables']) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 32a234b698f..abce0e8a11a 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -361,7 +361,6 @@ interface InferenceState { type CanonicalEdges = CanonicalWorkflow['edges'] -/** Appends `item` to the list `map` holds under `key`, starting the list when there is none. */ function pushTo(map: Map, key: K, item: V): void { const list = map.get(key) if (list) list.push(item) From b2db579e4761134bb7e75ce400f53852f6ed050a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:18:07 -0800 Subject: [PATCH 12/28] revert(workflow-code): keep binding a const swap by ident, as a display-name swap The swap of two consts is the same edit as a swap of two display names, which the apply intends as two renames (apply-workflow-source.integration.ts). Document the rule and the compile-side gap instead. --- packages/workflow-code/DESIGN.md | 7 +- packages/workflow-code/src/lock.ts | 6 +- .../workflow-code/src/materialize/identity.ts | 8 +- .../src/materialize/index.test.ts | 84 ++++++++----------- 4 files changed, 49 insertions(+), 56 deletions(-) diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index df8378eeb68..a2e5dc45e1a 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -195,7 +195,12 @@ key-keyed lock; `mergeSourceMetaFromLock` stores a write's hints back by id. 1. `lock-key`: the lock entry under the same key, when its id is in prev; 2. `lock-ident`: a lock entry whose key the file dropped but whose ident it still uses - (`MaterializeOptions.compiledIdents`), so editing only the display string keeps the id; + (`MaterializeOptions.compiledIdents`), so editing only the display string keeps the id. A key + whose ident now belongs to another lock entry binds that entry too, even when both keys + survive: two blocks that swap display names (equivalently, two consts that swap names) are two + renames, and each block keeps the id, webhook and schedules of its const. The plan lists both; + `lockIdentRenames`, the compile-side mirror, covers only dropped keys and so reads such a + swap by key; 3. `db-name`: the prev block with the same key (the lockless and stale-lock path); 4. `inferred`: exactly one unmatched prev block and one unmatched compiled block with the same type and parent and agreeing edge neighbourhoods, so renaming the variable and the display diff --git a/packages/workflow-code/src/lock.ts b/packages/workflow-code/src/lock.ts index 6955918c01b..4b36f9caa54 100644 --- a/packages/workflow-code/src/lock.ts +++ b/packages/workflow-code/src/lock.ts @@ -662,8 +662,10 @@ export interface SourceHints { /** * Lock keys the source renamed, old → new: a source block under a key `isStored` rejects whose * ident is that of exactly one lock entry, a stored block's, whose key is gone from the source. - * This is materialize's `lock-ident` identity rule, so a compile that pairs source blocks with - * stored ones by key agrees with materialize on which stored block a renamed one continues. + * This is materialize's `lock-ident` identity rule for a dropped key, so a compile that pairs + * source blocks with stored ones by key agrees with materialize on which stored block a renamed + * one continues. A swap where both keys survive, which materialize also binds by ident (DESIGN.md + * rule 2), is left out: the compile reads it by key. */ export function lockIdentRenames( hints: SourceHints, diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index abce0e8a11a..4fa68d44234 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -203,10 +203,8 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { const open = () => keys.filter((key) => !resolved.has(key) && !settled.has(key)) /** - * A key whose ident in the file now belongs to another lock entry whose key the file dropped: - * in a chain rename (A takes B's name while B takes a new one) the key is B's, but the block is - * A's, so rule 2 binds it. When the owner's key survives (two consts swapped), the key keeps its - * own entry, as rule 2 and `lockIdentRenames` read only a dropped key. + * A key whose ident in the file now belongs to another lock entry: in a chain rename (A takes + * B's name while B takes a new one) the key is B's, but the block is A's, so rule 2 binds it. */ const identOwner = new Map() for (const [key, entry] of Object.entries(lock?.blocks ?? {})) { @@ -217,7 +215,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { const entry = lockEntry(key) if (ident === undefined || !entry || entry.ident === ident) return false const owner = identOwner.get(ident) - return owner !== undefined && owner !== key && !Object.hasOwn(workflow.blocks, owner) + return owner !== undefined && owner !== key } /** Stale lock entries whose id left prev; reported only if no later rule matches the key. */ diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index a7c791ec7e1..16715e2e6a8 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -592,22 +592,6 @@ describe('materialize: identity', () => { expect(result.entries.filter((entry) => entry.status !== 'matched')).toEqual([]) }) - it('keeps each id by key when two blocks swap their consts and both keys survive', () => { - const prev = base() - const lock = lockFor(prev) - const result = run(canonical(prev, lock).workflow, lock, { - prev, - compiledIdents: { - ...Object.fromEntries(Object.entries(lock.blocks).map(([key, b]) => [key, b.ident])), - fetch: lock.blocks.report.ident, - report: lock.blocks.fetch.ident, - }, - }) - expect(result.identity.blocks.fetch.id).toBe(lock.blocks.fetch.id) - expect(result.identity.blocks.report.id).toBe(lock.blocks.report.id) - expect(result.entries.filter((entry) => entry.status !== 'matched')).toEqual([]) - }) - it('infers a lockless rename from type, parent and agreeing neighbours, flagged', () => { const prev = base() const result = run(renameFetch(canonical(prev).workflow), null, { prev }) @@ -1345,38 +1329,42 @@ describe('materialize: preserved UI state comes from prev', () => { expect(subBlockValue(result.state, 's1', 'text')).toBe('saw ') }) - it('rewrites a restored inactive value through a chain of renames at once', () => { - const state = workflowState([ - blockState('a1', 'function', 'Fetch', { code: 'return 1' }), - blockState('a2', 'function', 'Report', { code: 'return 2' }), - blockState('s1', 'slack', 'Post', { - operation: 'read', - channel: 'C1', - text: 'saw and ', - }), - ]) - const lock = lockFor(state) - const renamed: Record = { fetch: 'report', report: 'summary' } - const target = edit(canonical(state, lock).workflow, (draft) => { - const blocks = { ...draft.blocks } - draft.blocks = omit(draft.blocks, ['fetch', 'report']) - for (const [from, to] of Object.entries(renamed)) { - draft.blocks[to] = { ...blocks[from], key: to, name: to[0].toUpperCase() + to.slice(1) } - } - draft.blocks.post.params = omit(draft.blocks.post.params, ['text']) - }) - const result = run(target, lock, { - prev: state, - compiledIdents: Object.fromEntries( - Object.entries(renamed).map(([from, to]) => [to, lock.blocks[from].ident]) - ), - }) - expect(result.identity.blocks[renamed.fetch].id).toBe('a1') - expect(result.identity.blocks[renamed.report].id).toBe('a2') - expect(subBlockValue(result.state, 's1', 'text')).toBe( - 'saw and ' - ) - }) + it.each([ + ['a chain', { fetch: 'report', report: 'summary' }, 'saw and '], + ['a swap', { fetch: 'report', report: 'fetch' }, 'saw and '], + ])( + 'rewrites a restored inactive value through %s of renames at once', + (_label, moves, expected) => { + const state = workflowState([ + blockState('a1', 'function', 'Fetch', { code: 'return 1' }), + blockState('a2', 'function', 'Report', { code: 'return 2' }), + blockState('s1', 'slack', 'Post', { + operation: 'read', + channel: 'C1', + text: 'saw and ', + }), + ]) + const lock = lockFor(state) + const renamed: Record = moves + const target = edit(canonical(state, lock).workflow, (draft) => { + const blocks = { ...draft.blocks } + draft.blocks = omit(draft.blocks, ['fetch', 'report']) + for (const [from, to] of Object.entries(renamed)) { + draft.blocks[to] = { ...blocks[from], key: to, name: to[0].toUpperCase() + to.slice(1) } + } + draft.blocks.post.params = omit(draft.blocks.post.params, ['text']) + }) + const result = run(target, lock, { + prev: state, + compiledIdents: Object.fromEntries( + Object.entries(renamed).map(([from, to]) => [to, lock.blocks[from].ident]) + ), + }) + expect(result.identity.blocks[renamed.fetch].id).toBe('a1') + expect(result.identity.blocks[renamed.report].id).toBe('a2') + expect(subBlockValue(result.state, 's1', 'text')).toBe(expected) + } + ) it('clears a standalone advanced field that source emptied instead of restoring it', () => { const state = workflowState([ From cd954c01e057b282ebab0f5175766d3722350532 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:19:39 -0800 Subject: [PATCH 13/28] fix(workflow-code): seek reference candidates only among a group's new blocks --- .../workflow-code/src/materialize/identity.ts | 4 +++- .../workflow-code/src/materialize/index.test.ts | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 4fa68d44234..f41009aca8e 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -741,7 +741,9 @@ function inferRenames(state: InferenceState): void { const [first, ...rest] = renamedReferrers(prevKey).map(referencedByTarget) referenced.set( prevKey, - [...(first ?? [])].filter((key) => rest.every((keys) => keys.has(key))) + [...(first ?? [])].filter( + (key) => sides.targets.has(key) && rest.every((keys) => keys.has(key)) + ) ) } const referencing = invert(referenced) diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 16715e2e6a8..9a1b7cb31f2 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -676,6 +676,23 @@ describe('materialize: identity', () => { } ) + it('infers a rename by its renamed references beside a reference to no block', () => { + const prev = base() + const lock = lockFor(prev) + const target = edit(renameFetch(canonical(prev, lock).workflow), (draft) => { + draft.blocks.load.params.code = { kind: 'value', value: 'return 2' } + draft.blocks.report.params.code = { + kind: 'value', + value: 'return + ', + } + draft.edges = draft.edges.filter((e) => e.from !== 'start') + }) + const result = run(target, lock, { prev, compiledIdents: { load: 'x_load' } }) + expect(result.entries).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'load', id: 'a1', via: 'inferred' }) + ) + }) + it('infers a rename under a fresh lock when the ident changed too', () => { const prev = base() const lock = lockFor(prev) From 494d27b32749955822980f1753bd74aacbe2bd83 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:19:40 -0800 Subject: [PATCH 14/28] test(workflows): a dangling tag on a changed block is refused at compile, as check refuses it --- .../apply-workflow-source.integration.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 4a3abb65403..4fd3ba74049 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1628,7 +1628,7 @@ describe('source apply edge cases', () => { ) check( - 'a structural lint finding refuses only a block the apply changed; an untouched one warns', + 'a dangling reference refuses only a block the apply changed, at compile as an offline check does', async () => { const start = seedBlock('start_trigger', 'Start') const score = seedBlock('function', 'Score', { code: ['code', 'return 1'] }) @@ -1639,15 +1639,13 @@ describe('source apply edge cases', () => { const error = await expectRejected( apply(source, edit(source.files, 'js`return 1`', "'return '")), - 'E_APPLY_LINT' + 'E_APPLY_UNRESOLVED_REF' ) - const lint = error.diagnostics.filter((d) => d.code.endsWith('_APPLY_LINT')) - const refusedBlocks = lint.filter((d) => d.code === 'E_APPLY_LINT').map((d) => d.blockId) - expect(new Set(refusedBlocks)).toEqual(new Set([score.id])) - expect(lint).toContainEqual( - expect.objectContaining({ code: 'W_APPLY_LINT', blockId: other.id }) - ) + const refused = error.diagnostics.filter((d) => d.severity === 'error') + expect(refused.map((d) => [d.code, d.blockKey])).toEqual([ + ['E_APPLY_UNRESOLVED_REF', 'score'], + ]) expect(await snapshot(workflowId)).toEqual(before) } ) From 75ab54b9a34f590c626a48c44c242960bae008a7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:52:12 -0800 Subject: [PATCH 15/28] test(workflow-code): time the lock parent chain below the cap, where linear and quadratic growth separate --- packages/workflow-code/src/lock.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index 148a2bb799a..b11d26350f4 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -192,6 +192,8 @@ describe('parseWorkflowLock', () => { }) it('checks a long parent chain in linear time', () => { + // 64 to 512 blocks: a chain at the cap outgrows the CPU cache, which pushes linear growth + // toward the bound while a quadratic walk's shrinks (11x against 42x there, 9x against 57x here). expectLinearGrowth((n) => { const blocks: Record = {} for (let i = 0; i < n; i += 1) { @@ -204,7 +206,7 @@ describe('parseWorkflowLock', () => { } const lock = { version: 1, blocks, variables: {} } return () => parseWorkflowLock(lock) - }, SOURCE_LIMITS.maxBlocks / 8) + }, 64) }) it('refuses more blocks or variables than a workflow may hold, before reading them', () => { From 64f1c639011fe89b6a4aa421c2c896118e536691 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:52:31 -0800 Subject: [PATCH 16/28] fix(utils): write an array hole as null in canonicalJson, as JSON.stringify does --- packages/utils/src/object.test.ts | 10 +++++++++- packages/utils/src/object.ts | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/utils/src/object.test.ts b/packages/utils/src/object.test.ts index a9872f2b047..17294851f7b 100644 --- a/packages/utils/src/object.test.ts +++ b/packages/utils/src/object.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { getValueAtPath, isPlainRecord, toArray } from './object.js' +import { canonicalJson, getValueAtPath, isPlainRecord, toArray } from './object.js' class Sample { value = 1 @@ -34,3 +34,11 @@ describe('toArray', () => { expect(toArray(null)).not.toBe(toArray(null)) }) }) + +describe('canonicalJson', () => { + it('writes an array hole as null, as JSON.stringify does', () => { + const sparse: unknown[] = new Array(3) + sparse[1] = { b: 2, a: 1 } + expect(canonicalJson(sparse)).toBe('[null,{"a":1,"b":2},null]') + }) +}) diff --git a/packages/utils/src/object.ts b/packages/utils/src/object.ts index 195465ef93c..2b8ba0a968c 100644 --- a/packages/utils/src/object.ts +++ b/packages/utils/src/object.ts @@ -131,7 +131,7 @@ export function getValueAtPath(source: unknown, path: string): unknown { export function canonicalJson(value: unknown): string { if (value instanceof Date) return JSON.stringify(value.toISOString()) if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` + if (Array.isArray(value)) return `[${Array.from(value, canonicalJson).join(',')}]` const entries = Object.entries(value) .filter(([, entry]) => entry !== undefined) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) From 8ec20a1da96e9512f930beab5dadb48405be943e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:52:52 -0800 Subject: [PATCH 17/28] fix(workspace-source): stop a rows read at the row that passes the byte cap, not at the end of its page --- .../application/kinds/table/rows.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts index 3d88c6877ae..4b67aacb8a9 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts @@ -56,8 +56,9 @@ function sourceCell(column: ColumnDefinition, value: unknown): Json { /** * Reads every row of `table`: the cells of each column `keyById` names (column id → column key). * The version is read first, so rows written while the listing pages read as a server change on - * the next apply. The rows a rows file prints (those holding `rowKey`) are counted as they page - * in, and the read stops once they pass `maxRowsBytes`, which no directory could hold. + * the next apply. The rows a rows file prints (those holding `rowKey`) are counted as each one + * is read, and the read stops at the row that takes them past `maxRowsBytes`, which no directory + * could hold. */ export async function readStoredRows( ctx: SourceContext, @@ -94,15 +95,15 @@ export async function readStoredRows( const key = keyById.get(columnId) if (column && key !== undefined) cells[key] = sourceCell(column, value) } + if (hasRowKey(cells, rowKey)) printedBytes += printedRowBytes(cells) + if (printedBytes > WORKSPACE_SOURCE_LIMITS.maxRowsBytes) { + throw new OrchestrationError( + 'payload_too_large', + `The rows of table "${table.name}" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte limit a directory's rows files hold.` + ) + } data.push(cells) byColumnId.push(row.data) - if (hasRowKey(cells, rowKey)) printedBytes += printedRowBytes(cells) - } - if (printedBytes > WORKSPACE_SOURCE_LIMITS.maxRowsBytes) { - throw new OrchestrationError( - 'payload_too_large', - `The rows of table "${table.name}" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte limit a directory's rows files hold.` - ) } cursor = page.nextCursor ?? undefined } while (cursor) From 8cbf2ee6a3839c6f504b52cff79035cbe01c9037 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:55:11 -0800 Subject: [PATCH 18/28] fix(workflow-code): look up settings matches through a per-param list of the blocks lacking it, not a scan of every block --- .../workflow-code/src/materialize/identity.ts | 47 +------------ .../src/materialize/settings-index.test.ts | 33 +++++++++ .../src/materialize/settings-index.ts | 67 +++++++++++++++++++ 3 files changed, 101 insertions(+), 46 deletions(-) create mode 100644 packages/workflow-code/src/materialize/settings-index.test.ts create mode 100644 packages/workflow-code/src/materialize/settings-index.ts diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index f41009aca8e..7e9726ad55c 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -1,4 +1,3 @@ -import { canonicalJson } from '@sim/utils/object' import { findWorkflowReferenceTokens } from '@sim/utils/workflow-references' import { type BlockState, normalizeWorkflowBlockName } from '@sim/workflow-types/workflow' import type { CanonicalizeResult } from '../canonicalize' @@ -18,6 +17,7 @@ import { } from '../model' import { deepEqual, getOwn } from '../records' import type { MaterializeHooks } from './context' +import { settingsIndex } from './settings-index' /** How identity rule matched a compiled block to a stored one (rules 1–4). */ type IdentityMatchRule = 'lock-key' | 'lock-ident' | 'db-name' | 'inferred' @@ -457,51 +457,6 @@ function referencedKeys(block: CanonicalBlock): Set { return keys } -/** Whether any string under `value` holds a `<`, the only way a reference can sit in it. */ -function holdsAngle(value: unknown): boolean { - for (const text of strings(value)) if (text.includes('<')) return true - return false -} - -/** - * The blocks among `blocks` whose settings may agree with a given block's ({@link settingsAgree}), - * a superset found without comparing every pair, and the same whichever side is indexed. Settings - * agree only when every param both blocks hold is equal, so a match either lacks a param the given - * block holds or holds it equal. A value with no `<` holds no reference, so the old-key rewrite - * leaves it as it is and equal values have equal canonical JSON; a value with one keeps one after - * the rewrite, so it can only equal another such value. - */ -function settingsIndex(blocks: ReadonlyArray<[BlockKey, CanonicalBlock]>) { - const byValue = new Map() - const holders = new Map>() - const bucketOf = (id: string, value: unknown) => - holdsAngle(value) ? `<\u0000${id}` : `=\u0000${id}\u0000${canonicalJson(value)}` - for (const [key, block] of blocks) { - for (const [id, value] of Object.entries(block.params)) { - pushTo(byValue, bucketOf(id, value), key) - const held = holders.get(id) - if (held) held.add(key) - else holders.set(id, new Set([key])) - } - } - return function* matches(block: CanonicalBlock): Generator { - let best: { id: string; equal: BlockKey[] } | undefined - let bestSize = Number.POSITIVE_INFINITY - for (const [id, value] of Object.entries(block.params)) { - const equal = byValue.get(bucketOf(id, value)) ?? [] - const size = blocks.length - (holders.get(id)?.size ?? 0) + equal.length - if (size < bestSize) { - best = { id, equal } - bestSize = size - } - } - if (!best) return - const held = holders.get(best.id) - for (const [key] of blocks) if (!held?.has(key)) yield key - yield* best.equal - } -} - /** * The first half (rounded up) of a block's bound neighbours in `order`. Two blocks share a * majority of their neighbours, the `connections` signal, only if these halves meet: the earliest diff --git a/packages/workflow-code/src/materialize/settings-index.test.ts b/packages/workflow-code/src/materialize/settings-index.test.ts new file mode 100644 index 00000000000..1ba613339d7 --- /dev/null +++ b/packages/workflow-code/src/materialize/settings-index.test.ts @@ -0,0 +1,33 @@ +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' +import { describe, it } from 'vitest' +import type { CanonicalBlock } from '../model' +import { settingsIndex } from './settings-index' + +/** `n` function blocks sharing a language, each with its own code, so none settles a match. */ +function functionBlocks( + n: number, + prefix: string +): Array<[string, Pick]> { + return Array.from({ length: n }, (_, i) => [ + `${prefix}${i}`, + { + params: { + language: { kind: 'value', value: 'javascript' }, + code: { kind: 'value', value: `return '${prefix}${i}'` }, + }, + }, + ]) +} + +describe('settingsIndex', () => { + it('looks up blocks that all hold the narrowing param, each with its own value, in linear time', () => { + expectLinearGrowth((n) => { + const indexed = functionBlocks(n, 'old') + const lookups = functionBlocks(n, 'new') + return () => { + const matches = settingsIndex(indexed) + for (const [, block] of lookups) Array.from(matches(block)) + } + }, 250) + }) +}) diff --git a/packages/workflow-code/src/materialize/settings-index.ts b/packages/workflow-code/src/materialize/settings-index.ts new file mode 100644 index 00000000000..9796ca33ff9 --- /dev/null +++ b/packages/workflow-code/src/materialize/settings-index.ts @@ -0,0 +1,67 @@ +import { canonicalJson } from '@sim/utils/object' +import type { BlockKey, CanonicalBlock } from '../model' + +/** Whether any string under `value` holds a `<`, the only way a reference can sit in it. */ +function holdsAngle(value: unknown): boolean { + if (typeof value === 'string') return value.includes('<') + if (Array.isArray(value)) return value.some(holdsAngle) + if (value !== null && typeof value === 'object') return Object.values(value).some(holdsAngle) + return false +} + +/** + * The blocks among `blocks` whose settings may agree with a given block's, a superset found + * without comparing every pair, and the same whichever side is indexed. Settings agree only when + * every param both blocks hold is equal (references to a renamed block's old key read as its new + * one), so a match either lacks a param the given block holds or holds it equal. A value with no + * `<` holds no reference, so the old-key rewrite leaves it as it is and equal values have equal + * canonical JSON; a value with one keeps one after the rewrite, so it can only equal another such + * value. + * + * A lookup walks only the param it narrows to: the blocks holding that param's value (its bucket) + * and the blocks lacking the param, each list built once. + */ +export function settingsIndex(blocks: ReadonlyArray<[BlockKey, Pick]>) { + const byValue = new Map() + const holders = new Map>() + const bucketOf = (id: string, value: unknown) => + holdsAngle(value) ? `<\u0000${id}` : `=\u0000${id}\u0000${canonicalJson(value)}` + for (const [key, block] of blocks) { + for (const [id, value] of Object.entries(block.params)) { + const bucket = bucketOf(id, value) + const equal = byValue.get(bucket) + if (equal) equal.push(key) + else byValue.set(bucket, [key]) + const held = holders.get(id) + if (held) held.add(key) + else holders.set(id, new Set([key])) + } + } + const lacking = new Map() + const lackingOf = (id: string): BlockKey[] => { + let keys = lacking.get(id) + if (!keys) { + const held = holders.get(id) + keys = held + ? blocks.flatMap(([key]) => (held.has(key) ? [] : [key])) + : blocks.map(([key]) => key) + lacking.set(id, keys) + } + return keys + } + return function* matches(block: Pick): Generator { + let best: { id: string; equal: BlockKey[] } | undefined + let bestSize = Number.POSITIVE_INFINITY + for (const [id, value] of Object.entries(block.params)) { + const equal = byValue.get(bucketOf(id, value)) ?? [] + const size = blocks.length - (holders.get(id)?.size ?? 0) + equal.length + if (size < bestSize) { + best = { id, equal } + bestSize = size + } + } + if (!best) return + yield* lackingOf(best.id) + yield* best.equal + } +} From 6cb241bbac1f3c0c3d9677ad0db46326af85aef3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 01:57:23 -0800 Subject: [PATCH 19/28] fix(workflow-code): warn on a new block of a type the catalog keeps only for existing blocks, refused on an edit as the placement gate refuses it --- apps/docs/content/docs/cli/reference.mdx | 4 +- apps/docs/content/docs/cli/workflows.mdx | 2 +- apps/docs/content/docs/cli/workspaces.mdx | 2 +- apps/docs/openapi-v2-workflows.json | 9 +++++ packages/sim-cli/src/generated/v2-api.ts | 9 +++++ packages/workflow-code/src/compile/blocks.ts | 25 ++++++++++-- .../workflow-code/src/diagnostics/codes.ts | 6 +++ .../src/diagnostics/refusals.test.ts | 38 +++++++++++++++++++ .../workflow-code/src/diagnostics/refusals.ts | 6 ++- 9 files changed, 92 insertions(+), 9 deletions(-) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 4e17d50caa6..50088771879 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -10018,7 +10018,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 288570691eb..5c38f081196 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 317c16219dc..b0277098106 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -787,7 +787,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 101dc9ca9f6..39fe1971f2f 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -19205,6 +19205,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -19677,6 +19678,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -20667,6 +20669,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -21633,6 +21636,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -26688,6 +26692,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27265,6 +27270,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27685,6 +27691,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28551,6 +28558,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28971,6 +28979,7 @@ "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index ce952435617..378615c0cf7 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -875,6 +875,7 @@ export type ApplyWorkflowSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1374,6 +1375,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1666,6 +1668,7 @@ export type ApplyWorkspaceSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -5338,6 +5341,7 @@ export type CreateWorkflowFromSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -10445,6 +10449,7 @@ export type GetWorkflowSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -11138,6 +11143,7 @@ export type GetWorkspaceSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14466,6 +14472,7 @@ export type PlanWorkflowSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14920,6 +14927,7 @@ export type PlanWorkspaceSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -15212,6 +15220,7 @@ export type PlanWorkspaceSourceResponse = { | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 764f191a869..52d8043ce6f 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1953,7 +1953,7 @@ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBloc if (decl.duplicate || decl.builder.kind === 'unknown') return null if (decl.builder.kind !== 'container') { checkLegacyType(ctx, decl) - checkUncataloguedType(ctx, decl) + checkUnplaceableType(ctx, decl) } return decl.builder.kind === 'container' ? compileContainer(ctx, decl) @@ -1970,12 +1970,29 @@ function lockBinds(ctx: CompileContext, decl: BlockDecl): boolean { ) } -/** `W_UNKNOWN_BLOCK_TYPE` for a new `block()` of a type the catalog does not hold. */ -function checkUncataloguedType(ctx: CompileContext, decl: BlockDecl): void { +/** + * A new block of a type an apply cannot place: `W_UNKNOWN_BLOCK_TYPE` for a `block()` of a type the + * catalog does not hold, `W_TYPE_NOT_ADDABLE` for one it keeps only for blocks already using it + * (`addable: false`). A block the stored workflow may continue stays quiet: one the lock binds, + * or any block of a type the stored workflow holds, since rename inference may bind it to one. + */ +function checkUnplaceableType(ctx: CompileContext, decl: BlockDecl): void { const { builder } = decl - if (builder.kind !== 'generic' || builder.block || lockBinds(ctx, decl)) return + if (builder.kind !== 'generic' && builder.kind !== 'catalog') return + const notAddable = builder.block?.addable === false + if (!notAddable && (builder.kind !== 'generic' || builder.block)) return + if (lockBinds(ctx, decl)) return const stored = Object.values(ctx.stored?.blocks ?? {}) if (stored.some((block) => block.type === builder.type)) return + if (notAddable) { + ctx.reporter.report( + 'W_TYPE_NOT_ADDABLE', + `${builder.type} can stay where a workflow already uses it, but an apply cannot add a new block of it.`, + rangeOf(ctx, decl.call), + { hint: 'Use a block type this workspace offers.', blockKey: decl.key } + ) + return + } ctx.reporter.report( 'W_UNKNOWN_BLOCK_TYPE', `${builder.type} is not a block type this workspace offers, so an apply cannot add the block.`, diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index c47104e4880..4b2b6574609 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -216,6 +216,12 @@ const COMPILE_CATALOG_CODES = [ * its type quietly, so a workflow saved with a retired type still round-trips. */ 'W_UNKNOWN_BLOCK_TYPE', + /** + * A new block of a type the catalog keeps only for blocks already using it (`addable: false`): + * the server refuses to place it (`E_APPLY_TYPE_NOT_ADDABLE`). A block the stored workflow or + * the lock may continue stays quiet, as for `W_UNKNOWN_BLOCK_TYPE`. + */ + 'W_TYPE_NOT_ADDABLE', /** * A schedule created from source with no `timezone`: the editor seeds the viewer's zone, but * a schedule with none runs in UTC. diff --git a/packages/workflow-code/src/diagnostics/refusals.test.ts b/packages/workflow-code/src/diagnostics/refusals.test.ts index b0f1d8805ed..0c9a3b25cfe 100644 --- a/packages/workflow-code/src/diagnostics/refusals.test.ts +++ b/packages/workflow-code/src/diagnostics/refusals.test.ts @@ -139,6 +139,44 @@ describe('refuseEditedFindings', () => { expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([]) }) + it('refuses a new block of a type the catalog keeps only for blocks already using it', () => { + const catalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + api: { ...TEST_CATALOG.blocks.api, addable: false as const }, + }, + } + const body = [ + START, + "const a = api('A', { url: 'x' })", + "const b = block('api', 'B', { url: 'x' })", + ] + + const added = compile(body, {}, { catalog }) + expect(errors(refuseEditedFindings(added, edited('a', 'b')))).toEqual([ + 'E_APPLY_TYPE_NOT_ADDABLE a', + 'E_APPLY_TYPE_NOT_ADDABLE b', + ]) + + const bound = compile( + body, + {}, + { + catalog, + storedUnread: { + version: 1, + blocks: { + a: { id: 'b-a', parent: null, ident: 'a', order: 0 }, + b: { id: 'b-b', parent: null, ident: 'b', order: 1 }, + }, + variables: {}, + }, + } + ) + expect(errors(refuseEditedFindings(bound, edited('a', 'b')))).toEqual([]) + }) + it('refuses an iteration field a shared code file reads for a block outside the loop', () => { const compiled = compile( [ diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index 1f5f429c145..fdae6194e2a 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -67,6 +67,7 @@ export type EditRefusalCode = | 'E_APPLY_CODE_FILE_REF_SYNTAX' | 'E_APPLY_INVALID_SUBBLOCK_FIELD' | 'E_APPLY_INVALID_BLOCK_TYPE' + | 'E_APPLY_TYPE_NOT_ADDABLE' /** What an edit changed, as the caller can tell it. */ export interface EditedBlocks { @@ -109,12 +110,14 @@ const PARAM_REFUSALS: ReadonlyMap = new Map([ /** * In `workflow.ts` on a block the edit changed: a path no output of the block has, or a literal * `` whose head names no block, which the server's lint refuses as a dangling reference, - * or a new block of a type the catalog does not hold, which its placement gate refuses. + * or a new block of a type the catalog does not hold or keeps only for blocks already using it, + * which its placement gate refuses. */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], ['W_UNRESOLVED_REF', 'E_APPLY_UNRESOLVED_REF'], ['W_UNKNOWN_BLOCK_TYPE', 'E_APPLY_INVALID_BLOCK_TYPE'], + ['W_TYPE_NOT_ADDABLE', 'E_APPLY_TYPE_NOT_ADDABLE'], ]) /** @@ -140,6 +143,7 @@ export const REFUSABLE_WARNING_CODES: ReadonlySet = new S 'W_UNRESOLVED_REF', 'W_CODE_FILE_REF_SYNTAX', 'W_UNKNOWN_BLOCK_TYPE', + 'W_TYPE_NOT_ADDABLE', ]) /** The rules' error codes with the warning each refuses, for help text. */ From b8c4515434bb232e8946101f0c8b1151d7f4a8bd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:00:02 -0800 Subject: [PATCH 20/28] fix(workflow-code): cap variables at SOURCE_LIMITS.maxVariables in compile, the get check and the lock, so a returned lock always applies back --- apps/docs/openapi-v2-workflows.json | 106 +++++------------- .../contracts/v2/openapi/workflow-source.ts | 2 +- .../application/read-workflow-source.ts | 10 +- .../lib/workflows/code/scale-budget.test.ts | 9 +- .../workflow-code/src/compile/hostile.test.ts | 20 ++++ packages/workflow-code/src/compile/index.ts | 10 ++ packages/workflow-code/src/compile/parse.ts | 3 + packages/workflow-code/src/limits.ts | 19 +++- packages/workflow-code/src/lock.test.ts | 7 +- packages/workflow-code/src/lock.ts | 12 +- 10 files changed, 98 insertions(+), 100 deletions(-) diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 39fe1971f2f..1e6bae2e164 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -5911,7 +5911,7 @@ "get": { "operationId": "getWorkflowSource", "summary": "Get workflow source", - "description": "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", + "description": "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks or 2000 variables, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", "x-sim-operation": "workflows.source.read", "x-oauth-scope": "api:read", "tags": ["Workflows"], @@ -18953,6 +18953,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -19065,18 +19066,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -19865,6 +19865,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -19977,18 +19978,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -20156,6 +20156,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -20268,18 +20269,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -20351,33 +20351,7 @@ "required": ["files", "lock", "base"], "additionalProperties": false, "title": "Apply workflow source body", - "description": "Edited source files with the lock and base they were read with.", - "examples": [ - { - "files": { - "workflow.ts": "// @sim-sdk 1\nimport { trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Support triage', () => {\n const start = trigger.start('Start')\n})\n" - }, - "lock": { - "version": 1, - "blocks": { - "start": { - "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "parent": null, - "ident": "start", - "order": 0 - } - }, - "variables": {} - }, - "base": { - "version": 1, - "workflowId": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", - "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", - "fingerprint": "sf1:9c1d4e7f0a3b6c9d2e5f8a1b4c7d0e3f6a9b2c5d8e1f4a7b0c3d6e9f2a5b8c1d", - "catalogHash": "3f0c1a9e5b7d2c4f6a8e0b1d3c5f7a9e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a" - } - } - ] + "description": "Edited source files with the lock and base they were read with." }, "PlanWorkflowSourceResponse": { "type": "object", @@ -20856,6 +20830,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -20968,18 +20943,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -21147,6 +21121,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -21259,18 +21234,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -21318,33 +21292,7 @@ "required": ["files", "lock", "base"], "additionalProperties": false, "title": "Plan workflow source body", - "description": "Edited source files with the lock and base they were read with.", - "examples": [ - { - "files": { - "workflow.ts": "// @sim-sdk 1\nimport { trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Support triage', () => {\n const start = trigger.start('Start')\n})\n" - }, - "lock": { - "version": 1, - "blocks": { - "start": { - "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "parent": null, - "ident": "start", - "order": 0 - } - }, - "variables": {} - }, - "base": { - "version": 1, - "workflowId": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", - "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", - "fingerprint": "sf1:9c1d4e7f0a3b6c9d2e5f8a1b4c7d0e3f6a9b2c5d8e1f4a7b0c3d6e9f2a5b8c1d", - "catalogHash": "3f0c1a9e5b7d2c4f6a8e0b1d3c5f7a9e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a" - } - } - ] + "description": "Edited source files with the lock and base they were read with." }, "CreateWorkflowFromSourceResponse": { "type": "object", @@ -21823,6 +21771,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -21935,18 +21884,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], diff --git a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts index 25ca2eb1e1e..eab446626c4 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts @@ -51,7 +51,7 @@ export const workflowSourceOpenApiRoutes = [ applicationOperation: workflowOperations.readSource, operationId: 'getWorkflowSource', summary: 'Get workflow source', - description: `Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its \`base\`. The lock binds each block key in the source to its stored id; \`base\` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to \`functions/*\` files unless \`functionCode\` is \`inline\`. A workflow too large to apply back as source (more than ${SOURCE_LIMITS.maxBlocks} blocks, over ${SOURCE_LIMITS.maxTotalBytes} bytes of files in all, or nested deeper than ${SOURCE_LIMITS.maxDepth} levels) returns \`413\` naming the measured size. This read records no audit event. \`HEAD\` checks access as \`GET\` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns \`403\` while workflows as code is not enabled.`, + description: `Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its \`base\`. The lock binds each block key in the source to its stored id; \`base\` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to \`functions/*\` files unless \`functionCode\` is \`inline\`. A workflow too large to apply back as source (more than ${SOURCE_LIMITS.maxBlocks} blocks or ${SOURCE_LIMITS.maxVariables} variables, over ${SOURCE_LIMITS.maxTotalBytes} bytes of files in all, or nested deeper than ${SOURCE_LIMITS.maxDepth} levels) returns \`413\` naming the measured size. This read records no audit event. \`HEAD\` checks access as \`GET\` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns \`403\` while workflows as code is not enabled.`, tags: ['Workflows'], errors: [...RESOURCE_ERRORS, 'PayloadTooLarge'], success: { description: 'The workflow draft as source.', headers: RATE_LIMIT_HEADERS }, diff --git a/apps/sim/lib/workflows/application/read-workflow-source.ts b/apps/sim/lib/workflows/application/read-workflow-source.ts index 4c2fa7b670e..ee66f0cd412 100644 --- a/apps/sim/lib/workflows/application/read-workflow-source.ts +++ b/apps/sim/lib/workflows/application/read-workflow-source.ts @@ -117,6 +117,8 @@ function describeExcess(excess: SourceLimitExcess): string { return `it has ${excess.actual} blocks, over the ${excess.max}-block limit` case 'maxEdges': return `it has ${excess.actual} connections, over the ${excess.max}-connection limit` + case 'maxVariables': + return `it has ${excess.actual} variables, over the ${excess.max}-variable limit` case 'maxFiles': return `it prints ${excess.actual} files, over the ${excess.max}-file limit` case 'maxTotalBytes': @@ -131,7 +133,7 @@ function describeExcess(excess: SourceLimitExcess): string { */ function assertFitsSourceLimits( source: Pick, - counts: { blocks: number; edges: number } + counts: { blocks: number; edges: number; variables: number } ): void { const excesses = sourceLimitExcesses(source.files, counts) if (excesses.length > 0) { @@ -240,7 +242,11 @@ async function printLoadedSource( ): Promise { const { scope, gate, sourceMeta, prefetched } = loaded const { state, meta } = loaded.stored - const counts = { blocks: Object.keys(state.blocks).length, edges: state.edges.length } + const counts = { + blocks: Object.keys(state.blocks).length, + edges: state.edges.length, + variables: Object.keys(state.variables ?? {}).length, + } assertFitsSourceLimits({ files: {}, entryDepth: 0 }, counts) return withStoredCanonical( { organizationId: scope.workspaceOrganizationId, gate, prefetched, stored: loaded.stored }, diff --git a/apps/sim/lib/workflows/code/scale-budget.test.ts b/apps/sim/lib/workflows/code/scale-budget.test.ts index a0942a059fc..fdad59f60a8 100644 --- a/apps/sim/lib/workflows/code/scale-budget.test.ts +++ b/apps/sim/lib/workflows/code/scale-budget.test.ts @@ -207,6 +207,7 @@ describe('workflow source at the block cap', () => { sourceLimitExcesses(files, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS, edges: workflow?.edges.length ?? 0, + variables: 0, }) ).toEqual([]) expect(elapsed).toBeLessThan(ROUND_TRIP_BUDGET_MS) @@ -216,13 +217,17 @@ describe('workflow source at the block cap', () => { const source = syntheticSource(MAX_WORKFLOW_GRAPH_BLOCKS + 1) const compiled = compileText([{ path: 'workflow.ts', text: source }]) expect(errorsOf(compiled).map((diagnostic) => diagnostic.code)).toEqual(['E_LIMIT_EXCEEDED']) - expect(sourceLimitExcesses({}, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS + 1, edges: 0 })).toEqual([ + expect( + sourceLimitExcesses({}, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS + 1, edges: 0, variables: 0 }) + ).toEqual([ { limit: 'maxBlocks', actual: MAX_WORKFLOW_GRAPH_BLOCKS + 1, max: SOURCE_LIMITS.maxBlocks }, ]) }) it('refuses a stored graph over the edge cap in the get check, matching compile', () => { - expect(sourceLimitExcesses({}, { blocks: 2, edges: MAX_WORKFLOW_GRAPH_EDGES + 1 })).toEqual([ + expect( + sourceLimitExcesses({}, { blocks: 2, edges: MAX_WORKFLOW_GRAPH_EDGES + 1, variables: 0 }) + ).toEqual([ { limit: 'maxEdges', actual: MAX_WORKFLOW_GRAPH_EDGES + 1, max: SOURCE_LIMITS.maxEdges }, ]) }) diff --git a/packages/workflow-code/src/compile/hostile.test.ts b/packages/workflow-code/src/compile/hostile.test.ts index ee8e63b80be..252512d56d0 100644 --- a/packages/workflow-code/src/compile/hostile.test.ts +++ b/packages/workflow-code/src/compile/hostile.test.ts @@ -3,6 +3,8 @@ import { afterEach, describe, expect, it } from 'vitest' import { codeFileReferences } from '../code-files/references' import { FINDING_LIMITS } from '../diagnostics/bound' import { MAX_ERRORS } from '../diagnostics/codes' +import { SOURCE_LIMITS, sourceLimitExcesses } from '../limits' +import { parseWorkflowLock } from '../lock' import { TEST_CATALOG } from './catalog.test-helpers' import type { CompileResult } from './index' import { compileWorkflow } from './index' @@ -498,6 +500,24 @@ describe('one set of limits for get and compile', () => { expect(sourceDepthOverflow(files, Number.POSITIVE_INFINITY)).not.toBeNull() }) + it('refuses more variables than a get prints or a lock holds', () => { + const declare = (count: number) => { + const entries = Array.from( + { length: count }, + (_, i) => `v${i}: { type: 'string', value: '' }` + ) + return wrap(" const a = api('A', { url: 'x' })", `{ variables: { ${entries.join(', ')} } }`) + } + const max = SOURCE_LIMITS.maxVariables + expect(errorCodes(run(declare(max)))).toEqual([]) + expect(errorCodes(run(declare(max + 1)))).toEqual(['E_LIMIT_EXCEEDED']) + expect(sourceLimitExcesses({}, { blocks: 1, edges: 0, variables: max + 1 })).toEqual([ + { limit: 'maxVariables', actual: max + 1, max }, + ]) + const variables = Object.fromEntries(Array.from({ length: max }, (_, i) => [`v${i}`, `id${i}`])) + expect(parseWorkflowLock({ version: 1, blocks: {}, variables }).diagnostics).toBeUndefined() + }) + it('skips regex literal bodies, so many regexes with brackets and quotes pass', () => { const lines = Array.from({ length: 250 }, (_, i) => [ diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index cf7336bedd1..86bf413eb0c 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -208,6 +208,16 @@ function compileVariables(ctx: CompileContext, node: Node): void { ) return } + const overLimit = node.properties[ctx.limits.maxVariables] + if (overLimit) { + ctx.reporter.report( + 'E_LIMIT_EXCEEDED', + `The workflow declares ${node.properties.length} variables; the limit is ${ctx.limits.maxVariables}.`, + rangeOf(ctx, overLimit), + { hint: 'Keep related values together in one object or array variable.' } + ) + return + } const entries = objectEntries(ctx, node) const addItem = ctx.comments.list(node.start ?? 0, node.end ?? 0, { owner: encodeAnchor({ kind: 'header' }), diff --git a/packages/workflow-code/src/compile/parse.ts b/packages/workflow-code/src/compile/parse.ts index da414a6c6d4..4114b08bad2 100644 --- a/packages/workflow-code/src/compile/parse.ts +++ b/packages/workflow-code/src/compile/parse.ts @@ -12,6 +12,8 @@ export interface CompileLimits { maxBlocks?: number /** Edges the wiring adds, implied subflow starts included. */ maxEdges?: number + /** Variables the workflow declares. */ + maxVariables?: number /** Bytes of the entry plus the distinct code files `codeFile()` references. */ maxTotalBytes?: number /** Nesting depth of the entry file, and bracket depth of each JavaScript code file. */ @@ -29,6 +31,7 @@ export interface CompileLimits { export const DEFAULT_LIMITS: Required = { maxBlocks: SOURCE_LIMITS.maxBlocks, maxEdges: SOURCE_LIMITS.maxEdges, + maxVariables: SOURCE_LIMITS.maxVariables, maxTotalBytes: SOURCE_LIMITS.maxTotalBytes, maxDepth: SOURCE_LIMITS.maxDepth, /** One code file per block. */ diff --git a/packages/workflow-code/src/limits.ts b/packages/workflow-code/src/limits.ts index dd55dc55e5a..f3a8033d215 100644 --- a/packages/workflow-code/src/limits.ts +++ b/packages/workflow-code/src/limits.ts @@ -19,6 +19,8 @@ export const SOURCE_LIMITS = { maxBlocks: 2_000, /** Edges in one workflow. Mirrors the v2 graph cap (`MAX_WORKFLOW_GRAPH_EDGES`). */ maxEdges: 10_000, + /** Variables in one workflow, so a lock holds at most this many variable entries. */ + maxVariables: 2_000, /** Files in one source: `workflow.ts` plus one code file per block. */ maxFiles: 2_001, /** UTF-8 bytes of every file together, the entry included. */ @@ -34,7 +36,7 @@ export const SOURCE_LIMITS = { /** * Body cap for a request carrying source files: every file at its limit, doubled for JSON string * escaping (quotes, backslashes, newlines and tabs each escape to two bytes), plus room for a lock - * of `maxBlocks` entries. + * of `maxBlocks` block and `maxVariables` variable entries. */ export const SOURCE_REQUEST_MAX_BYTES = 2 * SOURCE_LIMITS.maxTotalBytes + 2_000_000 @@ -78,7 +80,7 @@ export function isCodeFilePath(path: string): boolean { /** One size limit a source exceeds, with the measured value. */ export interface SourceLimitExcess { - limit: 'maxBlocks' | 'maxEdges' | 'maxFiles' | 'maxTotalBytes' + limit: 'maxBlocks' | 'maxEdges' | 'maxVariables' | 'maxFiles' | 'maxTotalBytes' actual: number max: number } @@ -103,12 +105,12 @@ export function utf8Bytes(text: string, cap = Number.POSITIVE_INFINITY): number } /** - * Every size limit a printed source of a workflow with `counts` blocks and edges exceeds, in limit - * order; empty when it fits. Depth is `sourceDepthOverflow`'s. + * Every size limit a printed source of a workflow with `counts` blocks, edges and variables + * exceeds, in limit order; empty when it fits. Depth is `sourceDepthOverflow`'s. */ export function sourceLimitExcesses( files: Readonly>, - counts: { blocks: number; edges: number } + counts: { blocks: number; edges: number; variables: number } ): SourceLimitExcess[] { const excesses: SourceLimitExcess[] = [] if (counts.blocks > SOURCE_LIMITS.maxBlocks) { @@ -117,6 +119,13 @@ export function sourceLimitExcesses( if (counts.edges > SOURCE_LIMITS.maxEdges) { excesses.push({ limit: 'maxEdges', actual: counts.edges, max: SOURCE_LIMITS.maxEdges }) } + if (counts.variables > SOURCE_LIMITS.maxVariables) { + excesses.push({ + limit: 'maxVariables', + actual: counts.variables, + max: SOURCE_LIMITS.maxVariables, + }) + } const paths = Object.keys(files) if (paths.length > SOURCE_LIMITS.maxFiles) { excesses.push({ limit: 'maxFiles', actual: paths.length, max: SOURCE_LIMITS.maxFiles }) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index b11d26350f4..b3f6fa7d2fd 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -210,14 +210,11 @@ describe('parseWorkflowLock', () => { }) it('refuses more blocks or variables than a workflow may hold, before reading them', () => { - const over = SOURCE_LIMITS.maxBlocks + 1 const blocks: Record = {} const variables: Record = {} // Each entry is malformed, so a reader that walked them would report far more than the cap. - for (let i = 0; i < over; i += 1) { - blocks[`b${i}`] = { id: i } - variables[`v${i}`] = '' - } + for (let i = 0; i <= SOURCE_LIMITS.maxBlocks; i += 1) blocks[`b${i}`] = { id: i } + for (let i = 0; i <= SOURCE_LIMITS.maxVariables; i += 1) variables[`v${i}`] = '' expect(codesFor({ version: 1, blocks, variables: {} })).toEqual(['E_LOCK_SHAPE /blocks']) expect(codesFor({ version: 1, blocks: {}, variables })).toEqual(['E_LOCK_SHAPE /variables']) }) diff --git a/packages/workflow-code/src/lock.ts b/packages/workflow-code/src/lock.ts index 4b36f9caa54..d698f2d8b78 100644 --- a/packages/workflow-code/src/lock.ts +++ b/packages/workflow-code/src/lock.ts @@ -490,15 +490,15 @@ class LockReader { } } - /** Whether `raw` holds more entries than a workflow may; reported so it is never read. */ - overCap(raw: unknown, at: readonly string[]): boolean { - if (!isRecordLike(raw) || Object.keys(raw).length <= SOURCE_LIMITS.maxBlocks) return false - this.report('E_LOCK_SHAPE', at, `Expected at most ${SOURCE_LIMITS.maxBlocks} entries.`) + /** Whether `raw` holds more than `max` entries, a workflow's cap; reported so it is never read. */ + overCap(raw: unknown, max: number, at: readonly string[]): boolean { + if (!isRecordLike(raw) || Object.keys(raw).length <= max) return false + this.report('E_LOCK_SHAPE', at, `Expected at most ${max} entries.`) return true } variables(raw: unknown): Record { - if (this.overCap(raw, ['variables'])) return {} + if (this.overCap(raw, SOURCE_LIMITS.maxVariables, ['variables'])) return {} const variables = this.idRecord(raw, ['variables']) for (const key of Object.keys(variables)) { if (!key || normalizeWorkflowBlockName(key) !== key) { @@ -634,7 +634,7 @@ export function parseWorkflowLock(input: unknown): ParseWorkflowLockResult { const blockEntries: Array<[BlockKey, WorkflowLockBlock]> = [] if (!isRecordLike(json.blocks)) { reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') - } else if (!reader.overCap(json.blocks, ['blocks'])) { + } else if (!reader.overCap(json.blocks, SOURCE_LIMITS.maxBlocks, ['blocks'])) { for (const [key, raw] of Object.entries(json.blocks)) { const block = reader.block(key, raw) if (block) blockEntries.push([key, block]) From 0b2ffa7db34fb4e3146c7d4b46dc4db7d29e4de2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:00:03 -0800 Subject: [PATCH 21/28] fix(api): count a lock's entries before validating them, so an oversized lock costs one key count --- .../lib/api/contracts/workflow-source.test.ts | 26 +++++++------- apps/sim/lib/api/contracts/workflow-source.ts | 36 +++++++++++-------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/apps/sim/lib/api/contracts/workflow-source.test.ts b/apps/sim/lib/api/contracts/workflow-source.test.ts index 576267f654d..6afd06beeb4 100644 --- a/apps/sim/lib/api/contracts/workflow-source.test.ts +++ b/apps/sim/lib/api/contracts/workflow-source.test.ts @@ -2,27 +2,29 @@ import { SOURCE_LIMITS } from '@sim/workflow-code/limits' import { describe, expect, it } from 'vitest' import { workflowLockSchema } from '@/lib/api/contracts/workflow-source' -function lockOf(blockCount: number, variableCount: number) { +function lockOf(blockCount: number, variableCount: number, valid = true) { const blocks: Record = {} - const variables: Record = {} + const variables: Record = {} for (let i = 0; i < blockCount; i += 1) { - blocks[`b${i}`] = { id: `id${i}`, parent: null, ident: `b${i}`, order: i } + blocks[`b${i}`] = valid ? { id: `id${i}`, parent: null, ident: `b${i}`, order: i } : { id: i } } - for (let i = 0; i < variableCount; i += 1) variables[`v${i}`] = `var${i}` + for (let i = 0; i < variableCount; i += 1) variables[`v${i}`] = valid ? `var${i}` : i return { version: 1, blocks, variables } } +function issuePaths(lock: unknown): string[] | undefined { + return workflowLockSchema.safeParse(lock).error?.issues.map((issue) => issue.path.join('.')) +} + describe('workflowLockSchema', () => { it('accepts a lock at the block and variable cap', () => { - const max = SOURCE_LIMITS.maxBlocks - expect(workflowLockSchema.safeParse(lockOf(max, max)).success).toBe(true) + const lock = lockOf(SOURCE_LIMITS.maxBlocks, SOURCE_LIMITS.maxVariables) + expect(workflowLockSchema.safeParse(lock).success).toBe(true) }) - it('refuses a lock with more blocks or variables than a workflow may hold', () => { - const over = SOURCE_LIMITS.maxBlocks + 1 - const blocks = workflowLockSchema.safeParse(lockOf(over, 0)) - const variables = workflowLockSchema.safeParse(lockOf(0, over)) - expect(blocks.error?.issues.map((issue) => issue.path.join('.'))).toEqual(['blocks']) - expect(variables.error?.issues.map((issue) => issue.path.join('.'))).toEqual(['variables']) + it('refuses a lock with more blocks or variables than a workflow may hold, before reading them', () => { + // Every entry is malformed, so a schema that validated them first would report each one. + expect(issuePaths(lockOf(SOURCE_LIMITS.maxBlocks + 1, 0, false))).toEqual(['blocks']) + expect(issuePaths(lockOf(0, SOURCE_LIMITS.maxVariables + 1, false))).toEqual(['variables']) }) }) diff --git a/apps/sim/lib/api/contracts/workflow-source.ts b/apps/sim/lib/api/contracts/workflow-source.ts index 91e0bdadb4c..c44227b4d8a 100644 --- a/apps/sim/lib/api/contracts/workflow-source.ts +++ b/apps/sim/lib/api/contracts/workflow-source.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import type { DefinedField, SourceBase, @@ -78,9 +79,21 @@ const lockLayoutSchema = withExtensions({ 'Stored canvas geometry; applied only to new blocks.' ) -/** A lock never binds more entries than a workflow may hold blocks (`parseWorkflowLock` agrees). */ -function atMostMaxBlocks(record: Record): boolean { - return Object.keys(record).length <= SOURCE_LIMITS.maxBlocks +/** + * A string-keyed record of at most `max` entries, a workflow's cap (`parseWorkflowLock` agrees). + * The entries are counted before any value is validated, so an oversized lock costs one key count + * rather than a walk of every entry. + */ +function cappedRecord(values: Value, max: number, field: string) { + return z.preprocess( + (input, ctx) => { + if (isRecordLike(input) && Object.keys(input).length > max) { + ctx.addIssue({ code: 'custom', message: `${field} cannot hold more than ${max} entries` }) + } + return input + }, + z.record(z.string(), values) + ) } const lockBlockSchema = withExtensions({ @@ -113,17 +126,12 @@ const lockBlockSchema = withExtensions({ export const workflowLockSchema = withExtensions({ version: z.literal(1).describe('Lock format version.'), - blocks: z - .record(z.string(), lockBlockSchema) - .refine(atMostMaxBlocks, `lock.blocks cannot hold more than ${SOURCE_LIMITS.maxBlocks} entries`) - .describe('Block key → identity.'), - variables: z - .record(z.string(), z.string()) - .refine( - atMostMaxBlocks, - `lock.variables cannot hold more than ${SOURCE_LIMITS.maxBlocks} entries` - ) - .describe('Variable key → variable id.'), + blocks: cappedRecord(lockBlockSchema, SOURCE_LIMITS.maxBlocks, 'lock.blocks').describe( + 'Block key → identity.' + ), + variables: cappedRecord(z.string(), SOURCE_LIMITS.maxVariables, 'lock.variables').describe( + 'Variable key → variable id.' + ), } satisfies ShapeOf).describe( 'Identity sidecar (`.sim/lock.json`): binds the keys in the source to stored ids.' ) From 7f409444beb56f557efa3d5e3d3f1a127a05598a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:00:17 -0800 Subject: [PATCH 22/28] docs(workflow-code): list the variable cap with the other source limits --- apps/docs/content/docs/workflows/code.mdx | 2 +- packages/workflow-code/DESIGN.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index e686757a492..e8c439e9e2d 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -220,7 +220,7 @@ Run `sim workflows source apply --dir --dry-run` in CI to cat - Function code is one file per block, with no imports between files. - A change anyone makes while you edit means reading the workflow again; there is no automatic merge. - Workflows run in Sim, not locally. `--dry-run` checks everything a save checks, without running the workflow. -- A workflow can have up to 2,000 blocks, and its source up to 4 MB across `workflow.ts` and its function files together. A larger workflow is refused when you read or save it, and the CLI exits with code `7`. +- A workflow can have up to 2,000 blocks and 2,000 variables, and its source up to 4 MB across `workflow.ts` and its function files together. A larger workflow is refused when you read or save it, and the CLI exits with code `7`. To keep the tables, knowledge bases, tools and skills a workflow uses in the same directory, see [Workspace as code](/workflows/workspace-code). diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index a2e5dc45e1a..65e7d51c018 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -532,6 +532,7 @@ can always be applied back unchanged: - 2,000 blocks (the v2 graph cap, `MAX_WORKFLOW_GRAPH_BLOCKS`); - 10,000 edges (the v2 graph cap, `MAX_WORKFLOW_GRAPH_EDGES`); +- 2,000 variables, so a lock holds at most that many variable entries; - 2,001 files: `workflow.ts` plus one code file per block, so files mode never overflows into inline code; - 4,000,000 UTF-8 bytes across every file together, the entry included. No cap applies to a single From d21a5937f32f9acfe037659376fa378abfd3ec35 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:02:12 -0800 Subject: [PATCH 23/28] refactor(workflow-code): read stored types and lock idents once per compile, and share pushTo and strings from records --- packages/workflow-code/src/compile/blocks.ts | 10 +++++----- packages/workflow-code/src/compile/context.ts | 4 ++++ packages/workflow-code/src/compile/index.ts | 7 ++++++- .../workflow-code/src/materialize/identity.ts | 17 +---------------- .../src/materialize/settings-index.ts | 10 +++------- packages/workflow-code/src/records.ts | 16 ++++++++++++++++ 6 files changed, 35 insertions(+), 29 deletions(-) diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 52d8043ce6f..4bb0b2d8805 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1966,7 +1966,7 @@ function lockBinds(ctx: CompileContext, decl: BlockDecl): boolean { return ( lock !== undefined && (Object.hasOwn(lock.blocks, decl.key) || - Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) + (decl.ident !== null && ctx.storedUnreadIdents.has(decl.ident))) ) } @@ -1980,10 +1980,10 @@ function checkUnplaceableType(ctx: CompileContext, decl: BlockDecl): void { const { builder } = decl if (builder.kind !== 'generic' && builder.kind !== 'catalog') return const notAddable = builder.block?.addable === false - if (!notAddable && (builder.kind !== 'generic' || builder.block)) return - if (lockBinds(ctx, decl)) return - const stored = Object.values(ctx.stored?.blocks ?? {}) - if (stored.some((block) => block.type === builder.type)) return + const uncatalogued = builder.kind === 'generic' && !builder.block + if (!(notAddable || uncatalogued) || ctx.storedTypes.has(builder.type) || lockBinds(ctx, decl)) { + return + } if (notAddable) { ctx.reporter.report( 'W_TYPE_NOT_ADDABLE', diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index 21bd885d28d..c00a247215d 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -163,6 +163,10 @@ export interface CompileContext { * check continues without reading it. */ storedUnread: WorkflowLockV1 | undefined + /** The block types `stored` holds, read once so a per-block check stays constant time. */ + storedTypes: ReadonlySet + /** The idents `storedUnread` binds, read once so a per-block check stays constant time. */ + storedUnreadIdents: ReadonlySet /** `CompileInput.envNames`: the env names a run resolves, when known. */ envNames: ReadonlySet | undefined /** `CompileInput.envDeclaredIn`: the file the env names are declared in, when there is one. */ diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index 86bf413eb0c..506617fa696 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -560,6 +560,7 @@ function compileEntry( if (!ast) return emptyResult(reporter, parsed.sdkMajor) checkStorableLiterals(file, text, ast, reporter) + const storedUnread = input.stored === undefined ? input.storedUnread : undefined const ctx: CompileContext = { file, reporter, @@ -579,7 +580,11 @@ function compileEntry( metaSpans: {}, knownKeys: input.knownKeys ?? new Set(), stored: input.stored, - storedUnread: input.stored === undefined ? input.storedUnread : undefined, + storedUnread, + storedTypes: new Set(Object.values(input.stored?.blocks ?? {}).map((block) => block.type)), + storedUnreadIdents: new Set( + Object.values(storedUnread?.blocks ?? {}).map((entry) => entry.ident) + ), envNames: input.envNames, envDeclaredIn: input.envDeclaredIn, envReferences: new Set(), diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 7e9726ad55c..12558890949 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -15,7 +15,7 @@ import { isSimNode, type VarKey, } from '../model' -import { deepEqual, getOwn } from '../records' +import { deepEqual, getOwn, pushTo, strings } from '../records' import type { MaterializeHooks } from './context' import { settingsIndex } from './settings-index' @@ -359,12 +359,6 @@ interface InferenceState { type CanonicalEdges = CanonicalWorkflow['edges'] -function pushTo(map: Map, key: K, item: V): void { - const list = map.get(key) - if (list) list.push(item) - else map.set(key, [item]) -} - /** Each block's edges, in or out, so a block's neighbourhood reads only its own edges. */ function edgesByEndpoint(edges: CanonicalEdges): Map { const byKey = new Map() @@ -431,15 +425,6 @@ function settingsAgree( return shared } -/** Every string under `value`, depth first. */ -function* strings(value: unknown): Generator { - if (typeof value === 'string') yield value - else if (Array.isArray(value)) for (const item of value) yield* strings(item) - else if (value !== null && typeof value === 'object') { - for (const item of Object.values(value)) yield* strings(item) - } -} - /** * The keys a block references in any field a reference can sit in (its params, a condition or * router's branches, a loop or parallel's collection), each head read as the resolver reads it: diff --git a/packages/workflow-code/src/materialize/settings-index.ts b/packages/workflow-code/src/materialize/settings-index.ts index 9796ca33ff9..0589b0a57f5 100644 --- a/packages/workflow-code/src/materialize/settings-index.ts +++ b/packages/workflow-code/src/materialize/settings-index.ts @@ -1,11 +1,10 @@ import { canonicalJson } from '@sim/utils/object' import type { BlockKey, CanonicalBlock } from '../model' +import { pushTo, strings } from '../records' /** Whether any string under `value` holds a `<`, the only way a reference can sit in it. */ function holdsAngle(value: unknown): boolean { - if (typeof value === 'string') return value.includes('<') - if (Array.isArray(value)) return value.some(holdsAngle) - if (value !== null && typeof value === 'object') return Object.values(value).some(holdsAngle) + for (const text of strings(value)) if (text.includes('<')) return true return false } @@ -28,10 +27,7 @@ export function settingsIndex(blocks: ReadonlyArray<[BlockKey, Pick(map: Map, key: K, item: V): void { + const list = map.get(key) + if (list) list.push(item) + else map.set(key, [item]) +} + +/** Every string under `value`, depth first. */ +export function* strings(value: unknown): Generator { + if (typeof value === 'string') yield value + else if (Array.isArray(value)) for (const item of value) yield* strings(item) + else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) yield* strings(item) + } +} From 106fc07ed540923b0cccb2d16dc7c714b2640f4e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:06:21 -0800 Subject: [PATCH 24/28] test(workflow-code): keep the parent chain test's comment to why it stays below the cap At the cap the linear walk measured 11x against the quadratic walk's 42x; at 64 to 512 blocks, 9x against 57x. --- packages/workflow-code/src/lock.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index b3f6fa7d2fd..2cebc9fdb98 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -192,8 +192,8 @@ describe('parseWorkflowLock', () => { }) it('checks a long parent chain in linear time', () => { - // 64 to 512 blocks: a chain at the cap outgrows the CPU cache, which pushes linear growth - // toward the bound while a quadratic walk's shrinks (11x against 42x there, 9x against 57x here). + // 64 to 512 blocks, not the cap: a chain at the cap outgrows the CPU cache, which narrows the + // gap between linear and quadratic growth. expectLinearGrowth((n) => { const blocks: Record = {} for (let i = 0; i < n; i += 1) { From 607475baae139bca1dd0e2e2f397550e2906ef33 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:20:38 -0800 Subject: [PATCH 25/28] chore(api): regenerate the MCP operations for the variable cap in the get description --- apps/sim/lib/api/mcp/generated/v2-operations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/api/mcp/generated/v2-operations.ts b/apps/sim/lib/api/mcp/generated/v2-operations.ts index a2d923769f9..2a032a31641 100644 --- a/apps/sim/lib/api/mcp/generated/v2-operations.ts +++ b/apps/sim/lib/api/mcp/generated/v2-operations.ts @@ -1820,7 +1820,7 @@ export const V2_MCP_OPERATIONS = { contract: v2GetWorkflowSourceContract, summary: 'Get workflow source', description: - "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", + "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks or 2000 variables, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", handler: () => import('@/app/api/v2/workflows/[workflowId]/source/route').then((route) => route.GET), }, From 798fdfda9cdd030a0b262d308e998a94b0c21eea Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 02:39:28 -0800 Subject: [PATCH 26/28] fix(workflow-code): count a lock's blocks and variables before reading either record --- packages/workflow-code/src/lock.test.ts | 3 +++ packages/workflow-code/src/lock.ts | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index 2cebc9fdb98..51b4241f3a1 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -217,6 +217,9 @@ describe('parseWorkflowLock', () => { for (let i = 0; i <= SOURCE_LIMITS.maxVariables; i += 1) variables[`v${i}`] = '' expect(codesFor({ version: 1, blocks, variables: {} })).toEqual(['E_LOCK_SHAPE /blocks']) expect(codesFor({ version: 1, blocks: {}, variables })).toEqual(['E_LOCK_SHAPE /variables']) + // An oversized variables record is refused without walking the malformed blocks beside it. + const atCap = Object.fromEntries(Object.entries(blocks).slice(0, SOURCE_LIMITS.maxBlocks)) + expect(codesFor({ version: 1, blocks: atCap, variables })).toEqual(['E_LOCK_SHAPE /variables']) }) it('rejects branch keys that are not identifiers', () => { diff --git a/packages/workflow-code/src/lock.ts b/packages/workflow-code/src/lock.ts index d698f2d8b78..5b868ccad0d 100644 --- a/packages/workflow-code/src/lock.ts +++ b/packages/workflow-code/src/lock.ts @@ -498,7 +498,6 @@ class LockReader { } variables(raw: unknown): Record { - if (this.overCap(raw, SOURCE_LIMITS.maxVariables, ['variables'])) return {} const variables = this.idRecord(raw, ['variables']) for (const key of Object.keys(variables)) { if (!key || normalizeWorkflowBlockName(key) !== key) { @@ -630,11 +629,16 @@ export function parseWorkflowLock(input: unknown): ParseWorkflowLockResult { return { diagnostics: reader.diagnostics } } + // Both counts come before either record is read, so an oversized one never costs a walk of the other. + const blocksOverCap = reader.overCap(json.blocks, SOURCE_LIMITS.maxBlocks, ['blocks']) + const variablesOverCap = reader.overCap(json.variables, SOURCE_LIMITS.maxVariables, ['variables']) + if (blocksOverCap || variablesOverCap) return { diagnostics: reader.diagnostics } + const extensions = reader.extensions(json, LOCK_FIELDS, []) const blockEntries: Array<[BlockKey, WorkflowLockBlock]> = [] if (!isRecordLike(json.blocks)) { reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') - } else if (!reader.overCap(json.blocks, SOURCE_LIMITS.maxBlocks, ['blocks'])) { + } else { for (const [key, raw] of Object.entries(json.blocks)) { const block = reader.block(key, raw) if (block) blockEntries.push([key, block]) From de1126bcae27000c887fcc9474da66ed033ead38 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 03:08:51 -0800 Subject: [PATCH 27/28] fix(workspace-source): report a committed table write whose managed rows pass the source limits as saved After a table update commits, reading its managed rows back could pass the row or byte cap and fail the apply, leaving the caller on the old base though the edit was saved. The write now reprints the rows file as the directory declared it, keeps the base's rows entry, and warns (W_SERVER_CHANGED) that the rows were not read back. --- .../application/kinds/table/adapter.ts | 54 ++++++++++++++++--- .../application/kinds/table/rows.ts | 12 +++-- .../application/kinds/table/stored.ts | 9 ++-- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 598778c5679..97f5919437d 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -80,6 +80,7 @@ import { namedRows, readStoredRows, rowsVersion, + StoredRowsOverLimitError, unkeyedRows, } from '@/lib/workspace-source/application/kinds/table/rows' import { @@ -127,6 +128,11 @@ export interface TablePlan { changes?: TableChanges /** The base's column bindings the plan's fingerprint was read under. */ bindings?: Record + /** + * The base's rows entry, which the write records again when it cannot read the rows back + * (see `written`); absent when the base held none for this table. + */ + baseRows?: BaseResource['rows'] rows?: RowsPlan /** * The column keys the write records (`metadata.sourceColumnKeys`), by stored column name: each @@ -484,7 +490,8 @@ function withRows( ): TableChange { change.diagnostics.push(...rows.diagnostics) if (rows.plan) { - const bindings = entry?.id === state.table.id ? entry.columns : undefined + const based = entry?.id === state.table.id ? entry : undefined + const bindings = based?.columns return { ...change, action: 'update', @@ -493,7 +500,11 @@ function withRows( ? { destructive: { what: 'rows', counts: { rows: rows.removed } } } : {}), storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), - detail: { ...(bindings ? { bindings } : {}), rows: rows.plan }, + detail: { + ...(bindings ? { bindings } : {}), + ...(based?.rows ? { baseRows: based.rows } : {}), + rows: rows.plan, + }, } } if (!rows.base) return change @@ -696,6 +707,7 @@ async function planTable( detail: { changes, ...(bindings ? { bindings } : {}), + ...(entryOfStored?.rows ? { baseRows: entryOfStored.rows } : {}), ...(rows.plan ? { rows: rows.plan } : {}), }, }, @@ -750,19 +762,45 @@ function bindingsAfter(state: StoredTableState, decl: TableDecl): Record { const state = await reload(ctx, tableId) const bindings = bindingsAfter(state, decl) - const rows = await rowsToPrint(ctx, state, bindings, decl) - const { base, printed } = printStoredTable(state, key, bindings, rows) - return { id: tableId, base, text: printed.text } + const read = await rowsToPrint(ctx, state, bindings, decl).catch((error: unknown) => { + if (error instanceof StoredRowsOverLimitError) return error + throw error + }) + const overLimit = read instanceof StoredRowsOverLimitError + const { base, printed } = printStoredTable( + state, + key, + bindings, + overLimit ? { data: decl.rows?.data ?? [] } : read + ) + if (!overLimit) return { id: tableId, base, text: printed.text } + return { + id: tableId, + base: { ...base, ...(baseRows ? { rows: baseRows } : {}) }, + text: printed.text, + diagnostics: [ + kindFinding( + 'table', + key, + 'W_SERVER_CHANGED', + `${read.message} The apply did not read them back, so ${tableRowsPath(key)} stays as this directory sent it.`, + { hint: DRIFT_HINT } + ), + ], + } } /** The rows a stored table prints with: its managed rows read back, or the declaration's seed. */ @@ -955,7 +993,7 @@ export const tableAdapter: KindAdapter<'table', StoredTableState> = { throw new Error('A table update needs its plan') } await reshapeTableUseCase.execute({ principal, input: reshapeInput(ctx, tableId, plan) }) - return written(ctx, change.key, tableId, decl) + return written(ctx, change.key, tableId, decl, plan.baseRows) }, async read(stored, ctx) { diff --git a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts index 4b67aacb8a9..5e00b2e53ce 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts @@ -23,6 +23,12 @@ import { selectValueToNames } from '@/lib/table/select-values' import type { ColumnDefinition, RowData, TableDefinition } from '@/lib/table/types' import type { SourceContext } from '@/lib/workspace-source/application/types' +/** + * A managed table holds more rows, or more printed row bytes, than one directory's rows files may: + * a get skips the table, and a write already committed keeps the rows file the directory sent. + */ +export class StoredRowsOverLimitError extends OrchestrationError {} + /** A stored table's rows as source reads them. */ export interface StoredRows { /** Each row's source cells, keyed by column key. */ @@ -58,7 +64,7 @@ function sourceCell(column: ColumnDefinition, value: unknown): Json { * The version is read first, so rows written while the listing pages read as a server change on * the next apply. The rows a rows file prints (those holding `rowKey`) are counted as each one * is read, and the read stops at the row that takes them past `maxRowsBytes`, which no directory - * could hold. + * could hold ({@link StoredRowsOverLimitError}, as for a table past `maxTableRows`). */ export async function readStoredRows( ctx: SourceContext, @@ -67,7 +73,7 @@ export async function readStoredRows( rowKey: string ): Promise { if (table.rowCount > WORKSPACE_SOURCE_LIMITS.maxTableRows) { - throw new OrchestrationError( + throw new StoredRowsOverLimitError( 'validation', `Table "${table.name}" holds ${table.rowCount} rows; source manages at most ${WORKSPACE_SOURCE_LIMITS.maxTableRows}.` ) @@ -97,7 +103,7 @@ export async function readStoredRows( } if (hasRowKey(cells, rowKey)) printedBytes += printedRowBytes(cells) if (printedBytes > WORKSPACE_SOURCE_LIMITS.maxRowsBytes) { - throw new OrchestrationError( + throw new StoredRowsOverLimitError( 'payload_too_large', `The rows of table "${table.name}" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte limit a directory's rows files hold.` ) diff --git a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts index 81c109b6608..ea38639c3af 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts @@ -138,14 +138,15 @@ export function storedFingerprint( /** * The base entry and printed files of a stored table read under `bindings`. `rows` carries the - * rows to print: a managed table's stored rows with the row version they were read at, or the - * seed rows a declaration wrote, which the server keeps no record of. + * rows to print: a managed table's rows with the row version they were read at (without one, the + * base records no rows entry), or the seed rows a declaration wrote, which the server keeps no + * record of. */ export function printStoredTable( state: StoredTableState, key: string, bindings?: Readonly>, - rows?: { data: TableRowCells[]; version: number } | { seed: TableRowsDecl } + rows?: { data: TableRowCells[]; version?: number } | { seed: TableRowsDecl } ): { base: BaseResource; printed: PrintedDeclaration } { const { decl, keyById, fingerprint } = tableAsDecl(state, bindings) let rowsEntry: BaseRows | undefined @@ -153,7 +154,7 @@ export function printStoredTable( if (decl.rows?.mode !== 'managed') decl.rows = rows.seed } else if (decl.rows && rows) { decl.rows = { ...decl.rows, data: rows.data } - rowsEntry = baseRows(decl.rows, rows.version) + if (rows.version !== undefined) rowsEntry = baseRows(decl.rows, rows.version) } const columns: Record = {} for (const column of sourceColumns(state.table)) { From 83618459855b3af8114eab3ba35812ff1c3811bd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 21 Nov 2026 03:08:52 -0800 Subject: [PATCH 28/28] test(workspace-source): a table edit whose managed rows grew past the byte limit applies and reapplies --- .../application/rows.integration.ts | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 9f75505c97c..7dea2cd0b02 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -180,6 +180,15 @@ async function addUiRow(workspaceId: string, data: Record { const workspaceId = await newWorkspace() await managedPlans(workspaceId) - const filler = 'x'.repeat(370_000) - const rows = Math.ceil(WORKSPACE_SOURCE_LIMITS.maxRowsBytes / filler.length) + 1 - for (let index = 0; index < rows; index++) { - await addUiRow(workspaceId, { code: `big${index}-${filler}`, seats: index }) - } + await growPastRowsBytes(workspaceId) const read = await runner.get(workspaceId) @@ -835,6 +840,47 @@ export default defineTable({ ]) }) + check( + 'saves a table edit whose managed rows grew past the rows byte limit, keeping the rows file as sent', + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + await growPastRowsBytes(workspaceId) + const described = { + ...read.files, + 'tables/plans.ts': MANAGED.replace( + "name: 'plans',", + "name: 'plans',\n description: 'Seats'," + ), + } + + const saved = await apply(workspaceId, described, { base: read.base }) + + expect(saved.failed).toBeUndefined() + expect(saved.applied).toMatchObject([{ kind: 'table', key: 'plans', action: 'update' }]) + expect((await storedTable(workspaceId)).description).toBe('Seats') + expect(saved.files).toEqual(described) + const entry = saved.base.resources['table:plans'] + expect(entry?.fingerprint).not.toBe(read.base.resources['table:plans']?.fingerprint) + expect(entry?.rows).toEqual(read.base.resources['table:plans']?.rows) + expect( + saved.diagnostics + .filter((finding) => finding.code === 'W_SERVER_CHANGED') + .map((finding) => finding.message) + ).toContainEqual( + expect.stringContaining( + `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` + ) + ) + + // The saved base applies again as a no-op, keeping the server's rows. + const again = await apply(workspaceId, saved.files, { base: saved.base }) + expect(again.failed).toBeUndefined() + expect(again.applied).toEqual([]) + expect(codes(again.diagnostics, 'warning')).toEqual(['W_SERVER_CHANGED']) + } + ) + check('seeds a new table once, and only a new or empty one', async () => { const workspaceId = await newWorkspace() const files = { 'tables/plans.ts': SEED, 'tables/plans.rows.jsonl': ROWS }