diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index b91e32700b9..50088771879 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -8641,7 +8641,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### sim workflows source status @@ -10018,7 +10018,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 7e76fc8249d..5c38f081196 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -1445,7 +1445,7 @@ Compiles workflow.ts (and the function files it names) with the compiler the ser Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. ### Show the content of the changes in a workflow source directory diff --git a/apps/docs/content/docs/cli/workspaces.mdx b/apps/docs/content/docs/cli/workspaces.mdx index 64e39c047d7..b0277098106 100644 --- a/apps/docs/content/docs/cli/workspaces.mdx +++ b/apps/docs/content/docs/cli/workspaces.mdx @@ -787,7 +787,7 @@ Compiles every workflows/<key>/workflow.ts (and the function files each na Exit codes: 0 no errors; 1 errors (each printed as file:line:column with its frame and hint), or no cached catalog (run the get once). With --output json, stdout is one document that also lists every finding under diagnostics. -These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. +These warnings, which an apply saves as written, fail the check as errors on every block the lock does not bind (a block added since the get, or every block of a new workflow): W_REQUIRED_MISSING, W_UNKNOWN_OPTION, W_UNKNOWN_OUTPUT_PATH, W_VALUE_KIND, W_OUT_OF_RANGE, W_LITERAL_SECRET, W_TABLE_UNKNOWN_COLUMN, W_CHILD_INPUT_UNKNOWN, W_BRANCH_UNWIRED, W_UNRESOLVED_REF, W_REF_OUT_OF_SCOPE. --strict fails on them in every block, except W_BRANCH_UNWIRED on a block the lock binds (the stored workflow may hold it, and the server then stays quiet); --no-strict fails on none of them. In every mode, as the server refuses them: W_REF_OUT_OF_SCOPE is the error E_APPLY_REF_OUT_OF_SCOPE in any file of a block the edit added or changed; W_UNKNOWN_OPTION is the error E_APPLY_INVALID_SUBBLOCK_FIELD in workflow.ts on a block the edit added or changed; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in workflow.ts on a block the edit added or changed; W_UNKNOWN_BLOCK_TYPE is the error E_APPLY_INVALID_BLOCK_TYPE in workflow.ts on a block the edit added or changed; W_TYPE_NOT_ADDABLE is the error E_APPLY_TYPE_NOT_ADDABLE in workflow.ts on a block the edit added or changed; W_UNRESOLVED_REF is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_UNKNOWN_OUTPUT_PATH is the error E_APPLY_UNRESOLVED_REF in a function file changed since the get; W_CODE_FILE_REF_SYNTAX is the error E_APPLY_CODE_FILE_REF_SYNTAX in a function file changed since the get. A block counts as changed when the lock does not bind it, when the source moved it into another container, or when it compiles differently than in the source the last get or apply printed (.sim/stored-edges.json records a digest of each block). That last comparison needs the catalog the source was printed with in the cache: without it, a bound block that did not move counts as unchanged, so the server may refuse what the check passes. A $ref in a function file to a block the lock binds that the source no longer declares stays a warning, unless a warning cap left findings out: the server may infer that block was renamed and rewrite the reference, and refuses it otherwise. W_JSON_REF_UNTYPED is reported only on blocks the lock does not bind, and with --strict on every block. Exit 10 (busy, retry): another sim workspaces source command is using this directory (SOURCE_DIR_BUSY); nothing was read or written. Run this one again once it finished. diff --git a/apps/docs/content/docs/workflows/code.mdx b/apps/docs/content/docs/workflows/code.mdx index e686757a492..e8c439e9e2d 100644 --- a/apps/docs/content/docs/workflows/code.mdx +++ b/apps/docs/content/docs/workflows/code.mdx @@ -220,7 +220,7 @@ Run `sim workflows source apply --dir --dry-run` in CI to cat - Function code is one file per block, with no imports between files. - A change anyone makes while you edit means reading the workflow again; there is no automatic merge. - Workflows run in Sim, not locally. `--dry-run` checks everything a save checks, without running the workflow. -- A workflow can have up to 2,000 blocks, and its source up to 4 MB across `workflow.ts` and its function files together. A larger workflow is refused when you read or save it, and the CLI exits with code `7`. +- A workflow can have up to 2,000 blocks and 2,000 variables, and its source up to 4 MB across `workflow.ts` and its function files together. A larger workflow is refused when you read or save it, and the CLI exits with code `7`. To keep the tables, knowledge bases, tools and skills a workflow uses in the same directory, see [Workspace as code](/workflows/workspace-code). diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 785097e19cd..f82e43746e7 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -5911,7 +5911,7 @@ "get": { "operationId": "getWorkflowSource", "summary": "Get workflow source", - "description": "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", + "description": "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks or 2000 variables, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", "x-sim-operation": "workflows.source.read", "x-oauth-scope": "api:read", "tags": ["Workflows"], @@ -18953,6 +18953,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -19065,18 +19066,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -19204,6 +19204,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -19675,6 +19677,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -19861,6 +19865,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -19973,18 +19978,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -20152,6 +20156,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -20264,18 +20269,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -20347,33 +20351,7 @@ "required": ["files", "lock", "base"], "additionalProperties": false, "title": "Apply workflow source body", - "description": "Edited source files with the lock and base they were read with.", - "examples": [ - { - "files": { - "workflow.ts": "// @sim-sdk 1\nimport { trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Support triage', () => {\n const start = trigger.start('Start')\n})\n" - }, - "lock": { - "version": 1, - "blocks": { - "start": { - "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "parent": null, - "ident": "start", - "order": 0 - } - }, - "variables": {} - }, - "base": { - "version": 1, - "workflowId": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", - "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", - "fingerprint": "sf1:9c1d4e7f0a3b6c9d2e5f8a1b4c7d0e3f6a9b2c5d8e1f4a7b0c3d6e9f2a5b8c1d", - "catalogHash": "3f0c1a9e5b7d2c4f6a8e0b1d3c5f7a9e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a" - } - } - ] + "description": "Edited source files with the lock and base they were read with." }, "PlanWorkflowSourceResponse": { "type": "object", @@ -20664,6 +20642,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -20850,6 +20830,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -20962,18 +20943,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -21141,6 +21121,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -21253,18 +21234,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -21312,33 +21292,7 @@ "required": ["files", "lock", "base"], "additionalProperties": false, "title": "Plan workflow source body", - "description": "Edited source files with the lock and base they were read with.", - "examples": [ - { - "files": { - "workflow.ts": "// @sim-sdk 1\nimport { trigger, workflow } from '@sim/workflow-sdk'\n\nexport default workflow('Support triage', () => {\n const start = trigger.start('Start')\n})\n" - }, - "lock": { - "version": 1, - "blocks": { - "start": { - "id": "a3f1c0b2-7a44-4c1d-9d3a-2b8e5f0a1c77", - "parent": null, - "ident": "start", - "order": 0 - } - }, - "variables": {} - }, - "base": { - "version": 1, - "workflowId": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", - "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", - "fingerprint": "sf1:9c1d4e7f0a3b6c9d2e5f8a1b4c7d0e3f6a9b2c5d8e1f4a7b0c3d6e9f2a5b8c1d", - "catalogHash": "3f0c1a9e5b7d2c4f6a8e0b1d3c5f7a9e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a" - } - } - ] + "description": "Edited source files with the lock and base they were read with." }, "CreateWorkflowFromSourceResponse": { "type": "object", @@ -21629,6 +21583,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -21815,6 +21771,7 @@ "description": "Lock format version." }, "blocks": { + "description": "Block key → identity.", "type": "object", "propertyNames": { "type": "string" @@ -21927,18 +21884,17 @@ "description": "An `x-` extension field, kept verbatim." }, "description": "One block’s identity." - }, - "description": "Block key → identity." + } }, "variables": { + "description": "Variable key → variable id.", "type": "object", "propertyNames": { "type": "string" }, "additionalProperties": { "type": "string" - }, - "description": "Variable key → variable id." + } } }, "required": ["version", "blocks", "variables"], @@ -26683,6 +26639,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27259,6 +27217,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -27678,6 +27638,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28543,6 +28505,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", @@ -28962,6 +28926,8 @@ "E_UNKNOWN_OPERATION", "W_UNKNOWN_MODEL", "W_LEGACY_BLOCK_TYPE", + "W_UNKNOWN_BLOCK_TYPE", + "W_TYPE_NOT_ADDABLE", "I_SCHEDULE_UTC", "E_STORED_VALUE_MISSING", "W_VALUE_KIND", diff --git a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts index 25ca2eb1e1e..eab446626c4 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/workflow-source.ts @@ -51,7 +51,7 @@ export const workflowSourceOpenApiRoutes = [ applicationOperation: workflowOperations.readSource, operationId: 'getWorkflowSource', summary: 'Get workflow source', - description: `Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its \`base\`. The lock binds each block key in the source to its stored id; \`base\` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to \`functions/*\` files unless \`functionCode\` is \`inline\`. A workflow too large to apply back as source (more than ${SOURCE_LIMITS.maxBlocks} blocks, over ${SOURCE_LIMITS.maxTotalBytes} bytes of files in all, or nested deeper than ${SOURCE_LIMITS.maxDepth} levels) returns \`413\` naming the measured size. This read records no audit event. \`HEAD\` checks access as \`GET\` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns \`403\` while workflows as code is not enabled.`, + description: `Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its \`base\`. The lock binds each block key in the source to its stored id; \`base\` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to \`functions/*\` files unless \`functionCode\` is \`inline\`. A workflow too large to apply back as source (more than ${SOURCE_LIMITS.maxBlocks} blocks or ${SOURCE_LIMITS.maxVariables} variables, over ${SOURCE_LIMITS.maxTotalBytes} bytes of files in all, or nested deeper than ${SOURCE_LIMITS.maxDepth} levels) returns \`413\` naming the measured size. This read records no audit event. \`HEAD\` checks access as \`GET\` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns \`403\` while workflows as code is not enabled.`, tags: ['Workflows'], errors: [...RESOURCE_ERRORS, 'PayloadTooLarge'], success: { description: 'The workflow draft as source.', headers: RATE_LIMIT_HEADERS }, diff --git a/apps/sim/lib/api/contracts/workflow-source.test.ts b/apps/sim/lib/api/contracts/workflow-source.test.ts new file mode 100644 index 00000000000..6afd06beeb4 --- /dev/null +++ b/apps/sim/lib/api/contracts/workflow-source.test.ts @@ -0,0 +1,30 @@ +import { SOURCE_LIMITS } from '@sim/workflow-code/limits' +import { describe, expect, it } from 'vitest' +import { workflowLockSchema } from '@/lib/api/contracts/workflow-source' + +function lockOf(blockCount: number, variableCount: number, valid = true) { + const blocks: Record = {} + const variables: Record = {} + for (let i = 0; i < blockCount; i += 1) { + blocks[`b${i}`] = valid ? { id: `id${i}`, parent: null, ident: `b${i}`, order: i } : { id: i } + } + for (let i = 0; i < variableCount; i += 1) variables[`v${i}`] = valid ? `var${i}` : i + return { version: 1, blocks, variables } +} + +function issuePaths(lock: unknown): string[] | undefined { + return workflowLockSchema.safeParse(lock).error?.issues.map((issue) => issue.path.join('.')) +} + +describe('workflowLockSchema', () => { + it('accepts a lock at the block and variable cap', () => { + const lock = lockOf(SOURCE_LIMITS.maxBlocks, SOURCE_LIMITS.maxVariables) + expect(workflowLockSchema.safeParse(lock).success).toBe(true) + }) + + it('refuses a lock with more blocks or variables than a workflow may hold, before reading them', () => { + // Every entry is malformed, so a schema that validated them first would report each one. + expect(issuePaths(lockOf(SOURCE_LIMITS.maxBlocks + 1, 0, false))).toEqual(['blocks']) + expect(issuePaths(lockOf(0, SOURCE_LIMITS.maxVariables + 1, false))).toEqual(['variables']) + }) +}) diff --git a/apps/sim/lib/api/contracts/workflow-source.ts b/apps/sim/lib/api/contracts/workflow-source.ts index 720ad00ad6f..c44227b4d8a 100644 --- a/apps/sim/lib/api/contracts/workflow-source.ts +++ b/apps/sim/lib/api/contracts/workflow-source.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import type { DefinedField, SourceBase, @@ -5,6 +6,7 @@ import type { WorkflowLockV1, } from '@sim/workflow-code' import { DIAGNOSTIC_CODES } from '@sim/workflow-code/diagnostics/codes' +import { SOURCE_LIMITS } from '@sim/workflow-code/limits' import { z } from 'zod' import { defineRouteContract } from '@/lib/api/contracts/types' import { workflowIdParamsSchema } from '@/lib/api/contracts/workflows' @@ -77,6 +79,23 @@ const lockLayoutSchema = withExtensions({ 'Stored canvas geometry; applied only to new blocks.' ) +/** + * A string-keyed record of at most `max` entries, a workflow's cap (`parseWorkflowLock` agrees). + * The entries are counted before any value is validated, so an oversized lock costs one key count + * rather than a walk of every entry. + */ +function cappedRecord(values: Value, max: number, field: string) { + return z.preprocess( + (input, ctx) => { + if (isRecordLike(input) && Object.keys(input).length > max) { + ctx.addIssue({ code: 'custom', message: `${field} cannot hold more than ${max} entries` }) + } + return input + }, + z.record(z.string(), values) + ) +} + const lockBlockSchema = withExtensions({ id: z.string().describe('Block id.'), parent: z.string().nullable().describe('Key of the enclosing loop or parallel, or null.'), @@ -107,8 +126,12 @@ const lockBlockSchema = withExtensions({ export const workflowLockSchema = withExtensions({ version: z.literal(1).describe('Lock format version.'), - blocks: z.record(z.string(), lockBlockSchema).describe('Block key → identity.'), - variables: z.record(z.string(), z.string()).describe('Variable key → variable id.'), + blocks: cappedRecord(lockBlockSchema, SOURCE_LIMITS.maxBlocks, 'lock.blocks').describe( + 'Block key → identity.' + ), + variables: cappedRecord(z.string(), SOURCE_LIMITS.maxVariables, 'lock.variables').describe( + 'Variable key → variable id.' + ), } satisfies ShapeOf).describe( 'Identity sidecar (`.sim/lock.json`): binds the keys in the source to stored ids.' ) diff --git a/apps/sim/lib/api/cursor-binding.ts b/apps/sim/lib/api/cursor-binding.ts index 920cc72886e..2bb07433332 100644 --- a/apps/sim/lib/api/cursor-binding.ts +++ b/apps/sim/lib/api/cursor-binding.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto' -import { filterUndefined } from '@sim/utils/object' +import { canonicalJson, filterUndefined } from '@sim/utils/object' /** * Caller-facing message for a cursor replayed under different filters. Separate @@ -124,26 +124,6 @@ export function instantScopePart(raw: string | undefined): string | undefined { return Number.isNaN(parsed) ? raw : new Date(parsed).toISOString() } -/** - * Deterministic JSON: object keys sorted so two structurally equal values - * serialize identically regardless of the key order they arrived in, and - * `undefined` members dropped so an omitted param and an absent one agree. - * - * Array order is preserved, because an array is a sequence in the general case. - * A filter whose array is really a set must canonicalize it first — see - * {@link parseUnorderedList} and {@link unorderedScopeOf} — or equivalent - * queries fingerprint differently and a valid cursor is refused. - */ -export function canonicalJson(value: unknown): string { - if (value instanceof Date) return JSON.stringify(value.toISOString()) - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` - const entries = Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(',')}}` -} - /** * Fingerprint of a canonical form, short enough to sit inside an opaque token. * diff --git a/apps/sim/lib/api/mcp/generated/v2-operations.ts b/apps/sim/lib/api/mcp/generated/v2-operations.ts index a2d923769f9..2a032a31641 100644 --- a/apps/sim/lib/api/mcp/generated/v2-operations.ts +++ b/apps/sim/lib/api/mcp/generated/v2-operations.ts @@ -1820,7 +1820,7 @@ export const V2_MCP_OPERATIONS = { contract: v2GetWorkflowSourceContract, summary: 'Get workflow source', description: - "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", + "Read the saved draft as TypeScript source written against the workflow SDK, with its identity lock and its `base`. The lock binds each block key in the source to its stored id; `base` records the workflow, workspace, freshness fingerprint and block catalog the source was read at. The fingerprint ignores canvas positions, so moving blocks does not change it. Workspace resources such as tables and knowledge bases appear by their resource key: allocated from the resource's name the first time source names it, then never changed, so renaming the resource keeps its key. An id that matches no resource in the workspace prints as an unresolved reference. Function-block code goes to `functions/*` files unless `functionCode` is `inline`. A workflow too large to apply back as source (more than 2000 blocks or 2000 variables, over 4000000 bytes of files in all, or nested deeper than 200 levels) returns `413` naming the measured size. This read records no audit event. `HEAD` checks access as `GET` does and answers with no body, without reading the draft, so it never reports a too-large workflow. Returns `403` while workflows as code is not enabled.\n\nOAuth scope: `api:read`.", handler: () => import('@/app/api/v2/workflows/[workflowId]/source/route').then((route) => route.GET), }, diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 24bf5deaaf7..3623352902a 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -3,6 +3,7 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { mapWithConcurrency } from '@sim/utils/concurrency' +import { canonicalJson } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { z } from 'zod' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' @@ -16,7 +17,7 @@ import { nativeSearchQueriesSchema, workspaceSearchFiltersSchema, } from '@/lib/api/contracts/mothership-assistant-tools' -import { canonicalJson, fingerprint, instantScopePart } from '@/lib/api/cursor-binding' +import { fingerprint, instantScopePart } from '@/lib/api/cursor-binding' import { admitContent } from '@/lib/core/application' import { env } from '@/lib/core/config/env' import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts index fab00d25925..4e579a5d8f9 100644 --- a/apps/sim/lib/table/rows/cursor.ts +++ b/apps/sim/lib/table/rows/cursor.ts @@ -18,8 +18,8 @@ * {@link assertCursorQueryBinding}. */ -import { isRecordLike } from '@sim/utils/object' -import { canonicalJson, canonicalUnorderedArray, fingerprint } from '@/lib/api/cursor-binding' +import { canonicalJson, isRecordLike } from '@sim/utils/object' +import { canonicalUnorderedArray, fingerprint } from '@/lib/api/cursor-binding' import { TableQueryValidationError } from '@/lib/table/errors' import type { Filter, Sort, TablePredicate, TableRow, TableRowsCursor } from '@/lib/table/types' diff --git a/apps/sim/lib/table/rows/live-schema.ts b/apps/sim/lib/table/rows/live-schema.ts index d27f7fcc183..9331931abe0 100644 --- a/apps/sim/lib/table/rows/live-schema.ts +++ b/apps/sim/lib/table/rows/live-schema.ts @@ -11,9 +11,9 @@ * write that waited sees the change. */ +import { canonicalJson } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { sql } from 'drizzle-orm' -import { canonicalJson } from '@/lib/api/cursor-binding' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { DbTransaction } from '@/lib/table/planner' diff --git a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts index 4a3abb65403..4fd3ba74049 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-source.integration.ts @@ -1628,7 +1628,7 @@ describe('source apply edge cases', () => { ) check( - 'a structural lint finding refuses only a block the apply changed; an untouched one warns', + 'a dangling reference refuses only a block the apply changed, at compile as an offline check does', async () => { const start = seedBlock('start_trigger', 'Start') const score = seedBlock('function', 'Score', { code: ['code', 'return 1'] }) @@ -1639,15 +1639,13 @@ describe('source apply edge cases', () => { const error = await expectRejected( apply(source, edit(source.files, 'js`return 1`', "'return '")), - 'E_APPLY_LINT' + 'E_APPLY_UNRESOLVED_REF' ) - const lint = error.diagnostics.filter((d) => d.code.endsWith('_APPLY_LINT')) - const refusedBlocks = lint.filter((d) => d.code === 'E_APPLY_LINT').map((d) => d.blockId) - expect(new Set(refusedBlocks)).toEqual(new Set([score.id])) - expect(lint).toContainEqual( - expect.objectContaining({ code: 'W_APPLY_LINT', blockId: other.id }) - ) + const refused = error.diagnostics.filter((d) => d.severity === 'error') + expect(refused.map((d) => [d.code, d.blockKey])).toEqual([ + ['E_APPLY_UNRESOLVED_REF', 'score'], + ]) expect(await snapshot(workflowId)).toEqual(before) } ) diff --git a/apps/sim/lib/workflows/application/read-workflow-source.ts b/apps/sim/lib/workflows/application/read-workflow-source.ts index 4c2fa7b670e..ee66f0cd412 100644 --- a/apps/sim/lib/workflows/application/read-workflow-source.ts +++ b/apps/sim/lib/workflows/application/read-workflow-source.ts @@ -117,6 +117,8 @@ function describeExcess(excess: SourceLimitExcess): string { return `it has ${excess.actual} blocks, over the ${excess.max}-block limit` case 'maxEdges': return `it has ${excess.actual} connections, over the ${excess.max}-connection limit` + case 'maxVariables': + return `it has ${excess.actual} variables, over the ${excess.max}-variable limit` case 'maxFiles': return `it prints ${excess.actual} files, over the ${excess.max}-file limit` case 'maxTotalBytes': @@ -131,7 +133,7 @@ function describeExcess(excess: SourceLimitExcess): string { */ function assertFitsSourceLimits( source: Pick, - counts: { blocks: number; edges: number } + counts: { blocks: number; edges: number; variables: number } ): void { const excesses = sourceLimitExcesses(source.files, counts) if (excesses.length > 0) { @@ -240,7 +242,11 @@ async function printLoadedSource( ): Promise { const { scope, gate, sourceMeta, prefetched } = loaded const { state, meta } = loaded.stored - const counts = { blocks: Object.keys(state.blocks).length, edges: state.edges.length } + const counts = { + blocks: Object.keys(state.blocks).length, + edges: state.edges.length, + variables: Object.keys(state.variables ?? {}).length, + } assertFitsSourceLimits({ files: {}, entryDepth: 0 }, counts) return withStoredCanonical( { organizationId: scope.workspaceOrganizationId, gate, prefetched, stored: loaded.stored }, diff --git a/apps/sim/lib/workflows/code/scale-budget.test.ts b/apps/sim/lib/workflows/code/scale-budget.test.ts index a0942a059fc..fdad59f60a8 100644 --- a/apps/sim/lib/workflows/code/scale-budget.test.ts +++ b/apps/sim/lib/workflows/code/scale-budget.test.ts @@ -207,6 +207,7 @@ describe('workflow source at the block cap', () => { sourceLimitExcesses(files, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS, edges: workflow?.edges.length ?? 0, + variables: 0, }) ).toEqual([]) expect(elapsed).toBeLessThan(ROUND_TRIP_BUDGET_MS) @@ -216,13 +217,17 @@ describe('workflow source at the block cap', () => { const source = syntheticSource(MAX_WORKFLOW_GRAPH_BLOCKS + 1) const compiled = compileText([{ path: 'workflow.ts', text: source }]) expect(errorsOf(compiled).map((diagnostic) => diagnostic.code)).toEqual(['E_LIMIT_EXCEEDED']) - expect(sourceLimitExcesses({}, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS + 1, edges: 0 })).toEqual([ + expect( + sourceLimitExcesses({}, { blocks: MAX_WORKFLOW_GRAPH_BLOCKS + 1, edges: 0, variables: 0 }) + ).toEqual([ { limit: 'maxBlocks', actual: MAX_WORKFLOW_GRAPH_BLOCKS + 1, max: SOURCE_LIMITS.maxBlocks }, ]) }) it('refuses a stored graph over the edge cap in the get check, matching compile', () => { - expect(sourceLimitExcesses({}, { blocks: 2, edges: MAX_WORKFLOW_GRAPH_EDGES + 1 })).toEqual([ + expect( + sourceLimitExcesses({}, { blocks: 2, edges: MAX_WORKFLOW_GRAPH_EDGES + 1, variables: 0 }) + ).toEqual([ { limit: 'maxEdges', actual: MAX_WORKFLOW_GRAPH_EDGES + 1, max: SOURCE_LIMITS.maxEdges }, ]) }) diff --git a/apps/sim/lib/workspace-source/application/deployments.integration.ts b/apps/sim/lib/workspace-source/application/deployments.integration.ts index 6f645c8a56c..1091ccce697 100644 --- a/apps/sim/lib/workspace-source/application/deployments.integration.ts +++ b/apps/sim/lib/workspace-source/application/deployments.integration.ts @@ -32,7 +32,10 @@ import { replaceWorkflowChatDeployment } from '@/lib/chat-deployments/applicatio import { OrchestrationError } from '@/lib/core/orchestration/types' import { encryptSecret } from '@/lib/core/security/encryption' import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' -import { deployWorkflow } from '@/lib/workflows/application/deployments' +import { + deployWorkflow, + readWorkflowDeploymentStatus, +} from '@/lib/workflows/application/deployments' import { readWorkflowSource } from '@/lib/workflows/application/read-workflow-source' import { listWorkflowTriggers } from '@/lib/workflows/application/triggers' import { updateWorkflowPublicApi } from '@/lib/workflows/application/update-workflow-deployment-settings' @@ -1181,6 +1184,34 @@ describe('workspace source: deployments', () => { } ) + check( + 'reads the deployment state of only the workflows the directory declares or its base records', + async () => { + const { workspaceId, id, files, base } = await deployedWorkspace() + const others: Files = {} + for (const name of ['alpha', 'beta', 'gamma']) { + others[`workflows/${name}/workflow.ts`] = NIGHTLY.replace( + "workflow('Nightly'", + `workflow('${name}'` + ) + others[`deployments/${name}.ts`] = DRAFT + } + const deployedOthers = await apply(workspaceId, others) + expect(deployedOthers.failed).toBeUndefined() + + const reads = vi.spyOn(readWorkflowDeploymentStatus, 'execute') + try { + const plan = await apply(workspaceId, files, { base, kinds: ['deployment'], dryRun: true }) + expect(plan.failed).toBeUndefined() + expect(new Set(reads.mock.calls.map(([call]) => call.input.workflowId))).toEqual( + new Set([id]) + ) + } finally { + reads.mockRestore() + } + } + ) + check('refuses a deployment of a workflow the workspace does not hold', async () => { const workspaceId = await workspaces.create({ name: 'Deploy nothing', ownerId }) const refused = await refusal(apply(workspaceId, { 'deployments/missing.ts': DRAFT })) diff --git a/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts index 523d3877984..4ccf679ecc4 100644 --- a/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/deployment/adapter.ts @@ -89,6 +89,7 @@ import type { PlanContext, PlannedChange, SourceContext, + StoredResource, StoredSet, } from '@/lib/workspace-source/application/types' import { resolveIds } from '@/lib/workspaces/resource-keys' @@ -96,9 +97,15 @@ import { resolveIds } from '@/lib/workspaces/resource-keys' /** Deployed workflows per page while listing them. */ const LIST_PAGE = 200 -/** Deployments a load reads at once. */ +/** Deployments a plan or a get reads at once. */ const READ_CONCURRENCY = 4 +/** + * A listed deployed workflow's settings: `null` until read. A load lists deployed workflows only; + * a plan reads the settings of those it needs ({@link readNeeded}) and a get those it prints. + */ +type ListedDeployment = StoredDeployment | null + /** The base entry field naming the variable a chat password was last set from. */ const PASSWORD_ENV = 'x-passwordEnv' @@ -491,7 +498,7 @@ async function planLiveDraft( async function planDeployment( key: string, decl: DeploymentDecl, - stored: StoredSet, + stored: StoredSet, ctx: PlanContext ): Promise { const change: DeploymentChange = { @@ -507,7 +514,8 @@ async function planDeployment( planned.kind === 'workflow' && planned.key === key ) const creating = workflowChange?.action === 'create' - const deployed = stored.bound.get(key) + const listed = stored.bound.get(key) + const deployed = listed?.state ? { ...listed, state: listed.state } : undefined const workflowId = deployed?.id ?? workflowChange?.storedId ?? @@ -837,23 +845,22 @@ function listAllDeployedWorkflows(ctx: SourceContext) { }) } -export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { +export const deploymentAdapter: KindAdapter<'deployment', ListedDeployment> = { codec: deploymentCodec, async load(ctx, { allocate }) { const workflows = await listDeployedWorkflows(ctx) - const records = await mapWithConcurrency(workflows, READ_CONCURRENCY, async (workflow) => { - const state = await readDeployment(ctx, workflow.id) - return state ? [{ id: workflow.id, name: workflow.name, state }] : [] - }) - return bindStored(ctx, 'workflow', records.flat(), { allocate }) + const records = workflows.map(({ id, name }) => ({ id, name, state: null })) + return bindStored(ctx, 'workflow', records, { allocate }) }, - async plan(desired, stored, ctx) { + async plan(desired, listed, ctx) { + const stored = await readNeeded(listed, desired, ctx) const changes: DeploymentChange[] = [] for (const [key, decl] of desired) changes.push(await planDeployment(key, decl, stored, ctx)) const undeclared = planUndeclared('deployment', desired, stored, ctx, { - fingerprint: (record, base) => storedFingerprint(record.state, passwordEnvOf(base)), + fingerprint: (record, base) => + record.state ? storedFingerprint(record.state, passwordEnvOf(base)) : '', detail: (record, base) => ({ workflowId: record.id, deploy: false, @@ -866,7 +873,7 @@ export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { const refused = change.diagnostics.find( (finding) => finding.code === 'E_APPLY_DELETE_NOT_ALLOWED' ) - if (!record || (change.action !== 'delete' && !refused)) continue + if (!record?.state || (change.action !== 'delete' && !refused)) continue const counts = await stopsOnUndeploy(ctx, record.id, record.state) if (change.destructive) change.destructive = { what: 'deployment', counts } if (refused) { @@ -1005,26 +1012,64 @@ export const deploymentAdapter: KindAdapter<'deployment', StoredDeployment> = { } }, - async read(stored) { - const { state } = stored - // A draft that differs reads as pinned, so applying what a get wrote never publishes it. - const decl = declOf(state, { live: state.needsRedeployment ? 'pinned' : 'draft' }) - if (!decl) { - throw new OrchestrationError( - 'validation', - state.chat?.authType === null - ? 'Its chat gates access in a way source cannot name (public, password, emails or SSO).' - : 'Its chat answers with a block the draft no longer has, which source cannot name.' + read: readListed, + + async readMany(stored, ctx) { + const reads = await mapWithConcurrency(stored, READ_CONCURRENCY, async (resource) => { + const outcome = await readListed(resource, ctx).then( + (read) => ({ read }), + (error: unknown) => ({ error }) ) - } - return { - text: deploymentCodec.print(stored.key, decl).text, - blobs: [], - base: baseEntry(stored.id, state, undefined), - } + return [resource.key, outcome] as const + }) + return new Map(reads) }, } +/** + * The listed deployments a plan reads the settings of: those the directory declares and those the + * base records, which a plan may update, adopt or undeploy. The rest stay unread (`null`): the plan + * only says they are kept. A workflow undeployed since the listing drops out, as if never listed. + */ +async function readNeeded( + listed: StoredSet, + desired: ReadonlyMap, + ctx: PlanContext +): Promise> { + const bound = await mapWithConcurrency( + [...listed.bound], + READ_CONCURRENCY, + async ([key, record]) => { + const recorded = ctx.base?.resources[resourceAddress('deployment', key)]?.id === record.id + if (!desired.has(key) && !recorded) return [[key, record] as const] + const state = await readDeployment(ctx, record.id) + return state ? [[key, { ...record, state }] as const] : [] + } + ) + return { bound: new Map(bound.flat()), unbound: listed.unbound } +} + +/** A listed deployment as source, its settings read now. */ +async function readListed(stored: StoredResource, ctx: SourceContext) { + const state = await readDeployment(ctx, stored.id) + if (!state) throw new OrchestrationError('not_found', 'The workflow is no longer deployed.') + // A draft that differs reads as pinned, so applying what a get wrote never publishes it. + const decl = declOf(state, { live: state.needsRedeployment ? 'pinned' : 'draft' }) + if (!decl) { + throw new OrchestrationError( + 'validation', + state.chat?.authType === null + ? 'Its chat gates access in a way source cannot name (public, password, emails or SSO).' + : 'Its chat answers with a block the draft no longer has, which source cannot name.' + ) + } + return { + text: deploymentCodec.print(stored.key, decl).text, + blobs: [], + base: baseEntry(stored.id, state, undefined), + } +} + /** * Warns on every workflow this apply edits that is deployed while no deployment file in the * directory decides what happens to its live version (`W_DEPLOYED_DRAFT_CHANGED`): the edit diff --git a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts index 1b07e4359c8..97f5919437d 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/adapter.ts @@ -80,6 +80,7 @@ import { namedRows, readStoredRows, rowsVersion, + StoredRowsOverLimitError, unkeyedRows, } from '@/lib/workspace-source/application/kinds/table/rows' import { @@ -127,6 +128,11 @@ export interface TablePlan { changes?: TableChanges /** The base's column bindings the plan's fingerprint was read under. */ bindings?: Record + /** + * The base's rows entry, which the write records again when it cannot read the rows back + * (see `written`); absent when the base held none for this table. + */ + baseRows?: BaseResource['rows'] rows?: RowsPlan /** * The column keys the write records (`metadata.sourceColumnKeys`), by stored column name: each @@ -409,7 +415,7 @@ async function planRows( return outcome } const rowKey = rows.key ?? '' - const stored = await readStoredRows(ctx, state.table, keyById) + const stored = await readStoredRows(ctx, state.table, keyById, rowKey) const kept = unkeyedRows(stored, rowKey) if (kept.length > 0) { outcome.diagnostics.push(unkeyedFinding(key, state.table.name, rowKey, kept.length)) @@ -484,7 +490,8 @@ function withRows( ): TableChange { change.diagnostics.push(...rows.diagnostics) if (rows.plan) { - const bindings = entry?.id === state.table.id ? entry.columns : undefined + const based = entry?.id === state.table.id ? entry : undefined + const bindings = based?.columns return { ...change, action: 'update', @@ -493,7 +500,11 @@ function withRows( ? { destructive: { what: 'rows', counts: { rows: rows.removed } } } : {}), storedFingerprint: planFingerprint(storedFingerprint(state, bindings), rows.plan), - detail: { ...(bindings ? { bindings } : {}), rows: rows.plan }, + detail: { + ...(bindings ? { bindings } : {}), + ...(based?.rows ? { baseRows: based.rows } : {}), + rows: rows.plan, + }, } } if (!rows.base) return change @@ -696,6 +707,7 @@ async function planTable( detail: { changes, ...(bindings ? { bindings } : {}), + ...(entryOfStored?.rows ? { baseRows: entryOfStored.rows } : {}), ...(rows.plan ? { rows: rows.plan } : {}), }, }, @@ -750,19 +762,45 @@ function bindingsAfter(state: StoredTableState, decl: TableDecl): Record { const state = await reload(ctx, tableId) const bindings = bindingsAfter(state, decl) - const rows = await rowsToPrint(ctx, state, bindings, decl) - const { base, printed } = printStoredTable(state, key, bindings, rows) - return { id: tableId, base, text: printed.text } + const read = await rowsToPrint(ctx, state, bindings, decl).catch((error: unknown) => { + if (error instanceof StoredRowsOverLimitError) return error + throw error + }) + const overLimit = read instanceof StoredRowsOverLimitError + const { base, printed } = printStoredTable( + state, + key, + bindings, + overLimit ? { data: decl.rows?.data ?? [] } : read + ) + if (!overLimit) return { id: tableId, base, text: printed.text } + return { + id: tableId, + base: { ...base, ...(baseRows ? { rows: baseRows } : {}) }, + text: printed.text, + diagnostics: [ + kindFinding( + 'table', + key, + 'W_SERVER_CHANGED', + `${read.message} The apply did not read them back, so ${tableRowsPath(key)} stays as this directory sent it.`, + { hint: DRIFT_HINT } + ), + ], + } } /** The rows a stored table prints with: its managed rows read back, or the declaration's seed. */ @@ -774,7 +812,9 @@ async function rowsToPrint( ) { const { decl: storedDecl, keyById } = tableAsDecl(state, bindings) const rowKey = storedDecl.rows?.key - if (rowKey !== undefined) return { ...(await readStoredRows(ctx, state.table, keyById)), rowKey } + if (rowKey !== undefined) { + return { ...(await readStoredRows(ctx, state.table, keyById, rowKey)), rowKey } + } return decl?.rows?.mode === 'seed' ? { seed: decl.rows } : undefined } @@ -953,7 +993,7 @@ export const tableAdapter: KindAdapter<'table', StoredTableState> = { throw new Error('A table update needs its plan') } await reshapeTableUseCase.execute({ principal, input: reshapeInput(ctx, tableId, plan) }) - return written(ctx, change.key, tableId, decl) + return written(ctx, change.key, tableId, decl, plan.baseRows) }, async read(stored, ctx) { diff --git a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts index 42daa42e64f..5e00b2e53ce 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/rows.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/rows.ts @@ -8,6 +8,7 @@ import type { Json } from '@sim/workflow-code' import type { BaseRows } from '@sim/workspace-code' import { hasRowKey, + printedRowBytes, type TableRowCells, type TableRowsDecl, tableRowsFingerprint, @@ -22,6 +23,12 @@ import { selectValueToNames } from '@/lib/table/select-values' import type { ColumnDefinition, RowData, TableDefinition } from '@/lib/table/types' import type { SourceContext } from '@/lib/workspace-source/application/types' +/** + * A managed table holds more rows, or more printed row bytes, than one directory's rows files may: + * a get skips the table, and a write already committed keeps the rows file the directory sent. + */ +export class StoredRowsOverLimitError extends OrchestrationError {} + /** A stored table's rows as source reads them. */ export interface StoredRows { /** Each row's source cells, keyed by column key. */ @@ -55,15 +62,18 @@ function sourceCell(column: ColumnDefinition, value: unknown): Json { /** * Reads every row of `table`: the cells of each column `keyById` names (column id → column key). * The version is read first, so rows written while the listing pages read as a server change on - * the next apply. + * the next apply. The rows a rows file prints (those holding `rowKey`) are counted as each one + * is read, and the read stops at the row that takes them past `maxRowsBytes`, which no directory + * could hold ({@link StoredRowsOverLimitError}, as for a table past `maxTableRows`). */ export async function readStoredRows( ctx: SourceContext, table: TableDefinition, - keyById: ReadonlyMap + keyById: ReadonlyMap, + rowKey: string ): Promise { if (table.rowCount > WORKSPACE_SOURCE_LIMITS.maxTableRows) { - throw new OrchestrationError( + throw new StoredRowsOverLimitError( 'validation', `Table "${table.name}" holds ${table.rowCount} rows; source manages at most ${WORKSPACE_SOURCE_LIMITS.maxTableRows}.` ) @@ -72,6 +82,7 @@ export async function readStoredRows( const columnsById = new Map(table.schema.columns.map((column) => [getColumnId(column), column])) const data: TableRowCells[] = [] const byColumnId: RowData[] = [] + let printedBytes = 0 let cursor: string | undefined do { const page = await listTableRows.execute({ @@ -90,6 +101,13 @@ export async function readStoredRows( const key = keyById.get(columnId) if (column && key !== undefined) cells[key] = sourceCell(column, value) } + if (hasRowKey(cells, rowKey)) printedBytes += printedRowBytes(cells) + if (printedBytes > WORKSPACE_SOURCE_LIMITS.maxRowsBytes) { + throw new StoredRowsOverLimitError( + 'payload_too_large', + `The rows of table "${table.name}" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte limit a directory's rows files hold.` + ) + } data.push(cells) byColumnId.push(row.data) } diff --git a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts index 81c109b6608..ea38639c3af 100644 --- a/apps/sim/lib/workspace-source/application/kinds/table/stored.ts +++ b/apps/sim/lib/workspace-source/application/kinds/table/stored.ts @@ -138,14 +138,15 @@ export function storedFingerprint( /** * The base entry and printed files of a stored table read under `bindings`. `rows` carries the - * rows to print: a managed table's stored rows with the row version they were read at, or the - * seed rows a declaration wrote, which the server keeps no record of. + * rows to print: a managed table's rows with the row version they were read at (without one, the + * base records no rows entry), or the seed rows a declaration wrote, which the server keeps no + * record of. */ export function printStoredTable( state: StoredTableState, key: string, bindings?: Readonly>, - rows?: { data: TableRowCells[]; version: number } | { seed: TableRowsDecl } + rows?: { data: TableRowCells[]; version?: number } | { seed: TableRowsDecl } ): { base: BaseResource; printed: PrintedDeclaration } { const { decl, keyById, fingerprint } = tableAsDecl(state, bindings) let rowsEntry: BaseRows | undefined @@ -153,7 +154,7 @@ export function printStoredTable( if (decl.rows?.mode !== 'managed') decl.rows = rows.seed } else if (decl.rows && rows) { decl.rows = { ...decl.rows, data: rows.data } - rowsEntry = baseRows(decl.rows, rows.version) + if (rows.version !== undefined) rowsEntry = baseRows(decl.rows, rows.version) } const columns: Record = {} for (const column of sourceColumns(state.table)) { diff --git a/apps/sim/lib/workspace-source/application/rows.integration.ts b/apps/sim/lib/workspace-source/application/rows.integration.ts index 00b465709fc..7dea2cd0b02 100644 --- a/apps/sim/lib/workspace-source/application/rows.integration.ts +++ b/apps/sim/lib/workspace-source/application/rows.integration.ts @@ -180,6 +180,15 @@ async function addUiRow(workspaceId: string, data: Record { + const workspaceId = await newWorkspace() + await managedPlans(workspaceId) + await growPastRowsBytes(workspaceId) + + const read = await runner.get(workspaceId) + + expect(read.files['tables/plans.rows.jsonl']).toBeUndefined() + expect(read.skipped).toEqual([ + { + kind: 'table', + key: 'plans', + reason: expect.stringContaining( + `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` + ), + }, + ]) + }) + + check( + 'saves a table edit whose managed rows grew past the rows byte limit, keeping the rows file as sent', + async () => { + const workspaceId = await newWorkspace() + const read = await managedPlans(workspaceId) + await growPastRowsBytes(workspaceId) + const described = { + ...read.files, + 'tables/plans.ts': MANAGED.replace( + "name: 'plans',", + "name: 'plans',\n description: 'Seats'," + ), + } + + const saved = await apply(workspaceId, described, { base: read.base }) + + expect(saved.failed).toBeUndefined() + expect(saved.applied).toMatchObject([{ kind: 'table', key: 'plans', action: 'update' }]) + expect((await storedTable(workspaceId)).description).toBe('Seats') + expect(saved.files).toEqual(described) + const entry = saved.base.resources['table:plans'] + expect(entry?.fingerprint).not.toBe(read.base.resources['table:plans']?.fingerprint) + expect(entry?.rows).toEqual(read.base.resources['table:plans']?.rows) + expect( + saved.diagnostics + .filter((finding) => finding.code === 'W_SERVER_CHANGED') + .map((finding) => finding.message) + ).toContainEqual( + expect.stringContaining( + `rows of table "plans" pass the ${WORKSPACE_SOURCE_LIMITS.maxRowsBytes}-byte` + ) + ) + + // The saved base applies again as a no-op, keeping the server's rows. + const again = await apply(workspaceId, saved.files, { base: saved.base }) + expect(again.failed).toBeUndefined() + expect(again.applied).toEqual([]) + expect(codes(again.diagnostics, 'warning')).toEqual(['W_SERVER_CHANGED']) + } + ) + check('seeds a new table once, and only a new or empty one', async () => { const workspaceId = await newWorkspace() const files = { 'tables/plans.ts': SEED, 'tables/plans.rows.jsonl': ROWS } diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 47f56e72e70..378615c0cf7 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -874,6 +874,8 @@ export type ApplyWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1372,6 +1374,8 @@ export type ApplyWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -1663,6 +1667,8 @@ export type ApplyWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -5334,6 +5340,8 @@ export type CreateWorkflowFromSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -10440,6 +10448,8 @@ export type GetWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -11132,6 +11142,8 @@ export type GetWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14459,6 +14471,8 @@ export type PlanWorkflowSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -14912,6 +14926,8 @@ export type PlanWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' @@ -15203,6 +15219,8 @@ export type PlanWorkspaceSourceResponse = { | 'E_UNKNOWN_OPERATION' | 'W_UNKNOWN_MODEL' | 'W_LEGACY_BLOCK_TYPE' + | 'W_UNKNOWN_BLOCK_TYPE' + | 'W_TYPE_NOT_ADDABLE' | 'I_SCHEDULE_UTC' | 'E_STORED_VALUE_MISSING' | 'W_VALUE_KIND' diff --git a/packages/table-cells/src/rows.ts b/packages/table-cells/src/rows.ts index e3b15d5740a..b9a98420ddd 100644 --- a/packages/table-cells/src/rows.ts +++ b/packages/table-cells/src/rows.ts @@ -4,6 +4,7 @@ * batch that repeat an earlier row's unique value. The server's table writes and the CLI's offline * check of a workspace directory's rows both run these, so they agree on every row. */ +import { canonicalJson } from '@sim/utils/object' import { cellRuleOf } from './rules' import type { CellColumn, CellRow, CellValue } from './types' @@ -72,26 +73,12 @@ export function coerceRowCells( } } -/** - * Deterministic JSON: object keys sorted and `undefined` members dropped, so two structurally - * equal values serialize identically whatever their key order. - */ -function canonicalCellJson(value: unknown): string { - if (value instanceof Date) return JSON.stringify(value.toISOString()) - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(canonicalCellJson).join(',')}]` - const entries = Object.entries(value) - .filter(([, entry]) => entry !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalCellJson(entry)}`).join(',')}}` -} - /** * The key two unique-column values share when the unique check treats them as equal. Object keys * are sorted, since the check compares JSONB, where key order carries no meaning. */ export function uniqueValueKey(value: CellValue, column: CellColumn): string { - return canonicalCellJson(cellRuleOf(column).valueForEquality?.(value) ?? value) + return canonicalJson(cellRuleOf(column).valueForEquality?.(value) ?? value) } /** The errors of each refused row of a batch, by its index. */ diff --git a/packages/utils/src/object.test.ts b/packages/utils/src/object.test.ts index a9872f2b047..17294851f7b 100644 --- a/packages/utils/src/object.test.ts +++ b/packages/utils/src/object.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { getValueAtPath, isPlainRecord, toArray } from './object.js' +import { canonicalJson, getValueAtPath, isPlainRecord, toArray } from './object.js' class Sample { value = 1 @@ -34,3 +34,11 @@ describe('toArray', () => { expect(toArray(null)).not.toBe(toArray(null)) }) }) + +describe('canonicalJson', () => { + it('writes an array hole as null, as JSON.stringify does', () => { + const sparse: unknown[] = new Array(3) + sparse[1] = { b: 2, a: 1 } + expect(canonicalJson(sparse)).toBe('[null,{"a":1,"b":2},null]') + }) +}) diff --git a/packages/utils/src/object.ts b/packages/utils/src/object.ts index 20cb57c8779..2b8ba0a968c 100644 --- a/packages/utils/src/object.ts +++ b/packages/utils/src/object.ts @@ -121,3 +121,19 @@ export function getValueAtPath(source: unknown, path: string): unknown { } return cursor } + +/** + * Deterministic JSON: object keys sorted so two structurally equal values serialize identically + * whatever order their keys arrived in, `undefined` members dropped so an omitted member and an + * absent one agree, and a `Date` written as its ISO string. Array order is kept: an array is a + * sequence, so a caller whose array is really a set sorts it first. + */ +export function canonicalJson(value: unknown): string { + if (value instanceof Date) return JSON.stringify(value.toISOString()) + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' + if (Array.isArray(value)) return `[${Array.from(value, canonicalJson).join(',')}]` + const entries = Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(',')}}` +} diff --git a/packages/workflow-code/DESIGN.md b/packages/workflow-code/DESIGN.md index df8378eeb68..65e7d51c018 100644 --- a/packages/workflow-code/DESIGN.md +++ b/packages/workflow-code/DESIGN.md @@ -195,7 +195,12 @@ key-keyed lock; `mergeSourceMetaFromLock` stores a write's hints back by id. 1. `lock-key`: the lock entry under the same key, when its id is in prev; 2. `lock-ident`: a lock entry whose key the file dropped but whose ident it still uses - (`MaterializeOptions.compiledIdents`), so editing only the display string keeps the id; + (`MaterializeOptions.compiledIdents`), so editing only the display string keeps the id. A key + whose ident now belongs to another lock entry binds that entry too, even when both keys + survive: two blocks that swap display names (equivalently, two consts that swap names) are two + renames, and each block keeps the id, webhook and schedules of its const. The plan lists both; + `lockIdentRenames`, the compile-side mirror, covers only dropped keys and so reads such a + swap by key; 3. `db-name`: the prev block with the same key (the lockless and stale-lock path); 4. `inferred`: exactly one unmatched prev block and one unmatched compiled block with the same type and parent and agreeing edge neighbourhoods, so renaming the variable and the display @@ -527,6 +532,7 @@ can always be applied back unchanged: - 2,000 blocks (the v2 graph cap, `MAX_WORKFLOW_GRAPH_BLOCKS`); - 10,000 edges (the v2 graph cap, `MAX_WORKFLOW_GRAPH_EDGES`); +- 2,000 variables, so a lock holds at most that many variable entries; - 2,001 files: `workflow.ts` plus one code file per block, so files mode never overflows into inline code; - 4,000,000 UTF-8 bytes across every file together, the entry included. No cap applies to a single diff --git a/packages/workflow-code/src/compile/blocks.ts b/packages/workflow-code/src/compile/blocks.ts index 0e486dbb5c2..4bb0b2d8805 100644 --- a/packages/workflow-code/src/compile/blocks.ts +++ b/packages/workflow-code/src/compile/blocks.ts @@ -1951,12 +1951,61 @@ function compileContainer(ctx: CompileContext, decl: BlockDecl): CompiledBlock | /** Pass 2 for one declaration. `null` for declarations already reported as invalid. */ export function compileBlock(ctx: CompileContext, decl: BlockDecl): CompiledBlock | null { if (decl.duplicate || decl.builder.kind === 'unknown') return null - if (decl.builder.kind !== 'container') checkLegacyType(ctx, decl) + if (decl.builder.kind !== 'container') { + checkLegacyType(ctx, decl) + checkUnplaceableType(ctx, decl) + } return decl.builder.kind === 'container' ? compileContainer(ctx, decl) : compileRegularBlock(ctx, decl) } +/** Whether the lock of an unread stored workflow binds `decl`, by key or by ident. */ +function lockBinds(ctx: CompileContext, decl: BlockDecl): boolean { + const lock = ctx.storedUnread + return ( + lock !== undefined && + (Object.hasOwn(lock.blocks, decl.key) || + (decl.ident !== null && ctx.storedUnreadIdents.has(decl.ident))) + ) +} + +/** + * A new block of a type an apply cannot place: `W_UNKNOWN_BLOCK_TYPE` for a `block()` of a type the + * catalog does not hold, `W_TYPE_NOT_ADDABLE` for one it keeps only for blocks already using it + * (`addable: false`). A block the stored workflow may continue stays quiet: one the lock binds, + * or any block of a type the stored workflow holds, since rename inference may bind it to one. + */ +function checkUnplaceableType(ctx: CompileContext, decl: BlockDecl): void { + const { builder } = decl + if (builder.kind !== 'generic' && builder.kind !== 'catalog') return + const notAddable = builder.block?.addable === false + const uncatalogued = builder.kind === 'generic' && !builder.block + if (!(notAddable || uncatalogued) || ctx.storedTypes.has(builder.type) || lockBinds(ctx, decl)) { + return + } + if (notAddable) { + ctx.reporter.report( + 'W_TYPE_NOT_ADDABLE', + `${builder.type} can stay where a workflow already uses it, but an apply cannot add a new block of it.`, + rangeOf(ctx, decl.call), + { hint: 'Use a block type this workspace offers.', blockKey: decl.key } + ) + return + } + ctx.reporter.report( + 'W_UNKNOWN_BLOCK_TYPE', + `${builder.type} is not a block type this workspace offers, so an apply cannot add the block.`, + rangeOf(ctx, decl.call), + { + hint: () => + didYouMean(builder.type, Object.keys(ctx.catalog.blocks)) ?? + 'Use a block type from the catalog, or a builder such as api() or fn().', + blockKey: decl.key, + } + ) +} + /** * `W_LEGACY_BLOCK_TYPE` for a new block of a legacy or deprecated type, naming its replacement and * that type's builders. A block the stored workflow already holds under that type stays quiet; an @@ -1968,15 +2017,7 @@ function checkLegacyType(ctx: CompileContext, decl: BlockDecl): void { const sunset = block?.lifecycle?.sunset if (!block || !sunset) return const stored = ctx.stored ? getOwn(ctx.stored.blocks, decl.key) : undefined - if (stored?.type === block.type) return - const lock = ctx.storedUnread - if ( - lock && - (Object.hasOwn(lock.blocks, decl.key) || - Object.values(lock.blocks).some((entry) => entry.ident === decl.ident)) - ) { - return - } + if (stored?.type === block.type || lockBinds(ctx, decl)) return const replacement = sunset.replacedBy const namespaces = replacement ? [ diff --git a/packages/workflow-code/src/compile/context.ts b/packages/workflow-code/src/compile/context.ts index fbd8f89b3df..c00a247215d 100644 --- a/packages/workflow-code/src/compile/context.ts +++ b/packages/workflow-code/src/compile/context.ts @@ -163,6 +163,10 @@ export interface CompileContext { * check continues without reading it. */ storedUnread: WorkflowLockV1 | undefined + /** The block types `stored` holds, read once so a per-block check stays constant time. */ + storedTypes: ReadonlySet + /** The idents `storedUnread` binds, read once so a per-block check stays constant time. */ + storedUnreadIdents: ReadonlySet /** `CompileInput.envNames`: the env names a run resolves, when known. */ envNames: ReadonlySet | undefined /** `CompileInput.envDeclaredIn`: the file the env names are declared in, when there is one. */ @@ -180,6 +184,8 @@ export interface CompileContext { * written last (`checkDynamicOutputRefs`). */ dynamicOutputRefs: DynamicOutputRef[] + /** Literal tags whose head names no block, checked once renames are known (`checkUnresolvedTags`). */ + unresolvedTags: LiteralTag[] /** `x.error` references, checked once the edges are known (`checkErrorOutputRefs`). */ errorOutputRefs: Array> /** @@ -194,6 +200,15 @@ export interface CompileContext { valueConsts: Set } +/** A `` in literal text whose head is no declared block's key. */ +export interface LiteralTag { + token: string + segments: RefSegment[] + normalized: BlockKey + at: Range + blockKey: BlockKey +} + /** A reference path into a block whose outputs its `responseFormat` declares. */ interface DynamicOutputRef { at: Range diff --git a/packages/workflow-code/src/compile/ergonomics.test.ts b/packages/workflow-code/src/compile/ergonomics.test.ts index 73f32f8eee1..cf89e3c1ba8 100644 --- a/packages/workflow-code/src/compile/ergonomics.test.ts +++ b/packages/workflow-code/src/compile/ergonomics.test.ts @@ -476,17 +476,21 @@ describe('header, names and hand-written references', () => { }) it.each([ - ['a cased block name', "''", true], - ['a name that is no block', "''", true], - ['the normalized key of a block', "''", false], - ])('warns W_REF_NOT_NORMALIZED for %s: %s', (_label, literal, warns) => { + ['a cased block name', "''", 'W_REF_NOT_NORMALIZED'], + ['a name that is no block', "''", 'W_UNRESOLVED_REF'], + ['a bare tag that is no reference', "''", 'W_REF_NOT_NORMALIZED'], + ['the normalized key of a block', "''", undefined], + ])('warns for %s: %s', (_label, literal, warning) => { const result = run( source(` const a = api('A', { url: 'x' })\n api('B', { url: ${literal} })`) ) - expect(codes(result).includes('W_REF_NOT_NORMALIZED')).toBe(warns) + const found = codes(result).filter( + (code) => code === 'W_REF_NOT_NORMALIZED' || code === 'W_UNRESOLVED_REF' + ) + expect(found).toEqual(warning ? [warning] : []) if (literal === "''") { - const warning = result.diagnostics.find((d) => d.code === 'W_REF_NOT_NORMALIZED') - expect(warning?.hint).toContain('${a.data}') + const diagnostic = result.diagnostics.find((d) => d.code === 'W_REF_NOT_NORMALIZED') + expect(diagnostic?.hint).toContain('${a.data}') } }) }) diff --git a/packages/workflow-code/src/compile/hostile.test.ts b/packages/workflow-code/src/compile/hostile.test.ts index ee8e63b80be..252512d56d0 100644 --- a/packages/workflow-code/src/compile/hostile.test.ts +++ b/packages/workflow-code/src/compile/hostile.test.ts @@ -3,6 +3,8 @@ import { afterEach, describe, expect, it } from 'vitest' import { codeFileReferences } from '../code-files/references' import { FINDING_LIMITS } from '../diagnostics/bound' import { MAX_ERRORS } from '../diagnostics/codes' +import { SOURCE_LIMITS, sourceLimitExcesses } from '../limits' +import { parseWorkflowLock } from '../lock' import { TEST_CATALOG } from './catalog.test-helpers' import type { CompileResult } from './index' import { compileWorkflow } from './index' @@ -498,6 +500,24 @@ describe('one set of limits for get and compile', () => { expect(sourceDepthOverflow(files, Number.POSITIVE_INFINITY)).not.toBeNull() }) + it('refuses more variables than a get prints or a lock holds', () => { + const declare = (count: number) => { + const entries = Array.from( + { length: count }, + (_, i) => `v${i}: { type: 'string', value: '' }` + ) + return wrap(" const a = api('A', { url: 'x' })", `{ variables: { ${entries.join(', ')} } }`) + } + const max = SOURCE_LIMITS.maxVariables + expect(errorCodes(run(declare(max)))).toEqual([]) + expect(errorCodes(run(declare(max + 1)))).toEqual(['E_LIMIT_EXCEEDED']) + expect(sourceLimitExcesses({}, { blocks: 1, edges: 0, variables: max + 1 })).toEqual([ + { limit: 'maxVariables', actual: max + 1, max }, + ]) + const variables = Object.fromEntries(Array.from({ length: max }, (_, i) => [`v${i}`, `id${i}`])) + expect(parseWorkflowLock({ version: 1, blocks: {}, variables }).diagnostics).toBeUndefined() + }) + it('skips regex literal bodies, so many regexes with brackets and quotes pass', () => { const lines = Array.from({ length: 250 }, (_, i) => [ diff --git a/packages/workflow-code/src/compile/index.ts b/packages/workflow-code/src/compile/index.ts index dce5a31729c..506617fa696 100644 --- a/packages/workflow-code/src/compile/index.ts +++ b/packages/workflow-code/src/compile/index.ts @@ -42,6 +42,7 @@ import { checkCodeFileRefs, checkDynamicOutputRefs, checkErrorOutputRefs, + checkUnresolvedTags, compileJsonLiteral, objectEntries, } from './values' @@ -207,6 +208,16 @@ function compileVariables(ctx: CompileContext, node: Node): void { ) return } + const overLimit = node.properties[ctx.limits.maxVariables] + if (overLimit) { + ctx.reporter.report( + 'E_LIMIT_EXCEEDED', + `The workflow declares ${node.properties.length} variables; the limit is ${ctx.limits.maxVariables}.`, + rangeOf(ctx, overLimit), + { hint: 'Keep related values together in one object or array variable.' } + ) + return + } const entries = objectEntries(ctx, node) const addItem = ctx.comments.list(node.start ?? 0, node.end ?? 0, { owner: encodeAnchor({ kind: 'header' }), @@ -549,6 +560,7 @@ function compileEntry( if (!ast) return emptyResult(reporter, parsed.sdkMajor) checkStorableLiterals(file, text, ast, reporter) + const storedUnread = input.stored === undefined ? input.storedUnread : undefined const ctx: CompileContext = { file, reporter, @@ -568,12 +580,17 @@ function compileEntry( metaSpans: {}, knownKeys: input.knownKeys ?? new Set(), stored: input.stored, - storedUnread: input.stored === undefined ? input.storedUnread : undefined, + storedUnread, + storedTypes: new Set(Object.values(input.stored?.blocks ?? {}).map((block) => block.type)), + storedUnreadIdents: new Set( + Object.values(storedUnread?.blocks ?? {}).map((entry) => entry.ident) + ), envNames: input.envNames, envDeclaredIn: input.envDeclaredIn, envReferences: new Set(), jsonFields: [], dynamicOutputRefs: [], + unresolvedTags: [], errorOutputRefs: [], refusedBranches: new Map(), valueConsts: new Set(), @@ -692,6 +709,7 @@ function compileEntry( : new Map() const renamed = new Map([...unreadRenames, ...(input.renamedKeys ?? [])]) checkCodeFileRefs(ctx, renamed) + checkUnresolvedTags(ctx, renamed) const stored = storedView(input.stored, renamed) const refTypes: RefTypeScope = { diff --git a/packages/workflow-code/src/compile/parse.ts b/packages/workflow-code/src/compile/parse.ts index da414a6c6d4..4114b08bad2 100644 --- a/packages/workflow-code/src/compile/parse.ts +++ b/packages/workflow-code/src/compile/parse.ts @@ -12,6 +12,8 @@ export interface CompileLimits { maxBlocks?: number /** Edges the wiring adds, implied subflow starts included. */ maxEdges?: number + /** Variables the workflow declares. */ + maxVariables?: number /** Bytes of the entry plus the distinct code files `codeFile()` references. */ maxTotalBytes?: number /** Nesting depth of the entry file, and bracket depth of each JavaScript code file. */ @@ -29,6 +31,7 @@ export interface CompileLimits { export const DEFAULT_LIMITS: Required = { maxBlocks: SOURCE_LIMITS.maxBlocks, maxEdges: SOURCE_LIMITS.maxEdges, + maxVariables: SOURCE_LIMITS.maxVariables, maxTotalBytes: SOURCE_LIMITS.maxTotalBytes, maxDepth: SOURCE_LIMITS.maxDepth, /** One code file per block. */ diff --git a/packages/workflow-code/src/compile/scaling.test.ts b/packages/workflow-code/src/compile/scaling.test.ts index a1412a8874c..e0b62ea33cf 100644 --- a/packages/workflow-code/src/compile/scaling.test.ts +++ b/packages/workflow-code/src/compile/scaling.test.ts @@ -4,7 +4,44 @@ import { expectLinearGrowth } from '@sim/testing/helpers/scaling' import { describe, it } from 'vitest' +import { SOURCE_LIMITS } from '../limits' +import { TEST_CATALOG } from './catalog.test-helpers' import { stronglyConnected } from './edges' +import { compileWorkflow } from './index' + +const ENTRY = 'workflows/test.ts' + +/** + * `n` api blocks and a function whose code file reads `refs` references `reference(i)` writes, + * nearly all of them findings past the warning caps. + */ +function compileReading( + n: number, + refs: number, + reference: (i: number) => string, + context: { knownKeys?: ReadonlySet; envNames?: ReadonlySet } +) { + const blocks = Array.from({ length: n }, (_, i) => ` const a${i} = api('A${i}', { url: 'x' })`) + const text = [ + '// @sim-sdk 1', + "import { api, codeFile, fn, trigger, workflow } from '@sim/workflow-sdk'", + "export default workflow('Test', {}, () => {", + " const start = trigger.start('Start', {})", + ...blocks, + " const f = fn('F', { code: codeFile('./functions/f.js') })", + '})', + '', + ].join('\n') + const code = Array.from({ length: refs }, (_, i) => `${reference(i)}\n`).join('') + return () => + compileWorkflow({ + entry: { path: ENTRY, text }, + files: new Map([['workflows/functions/f.js', code]]), + catalog: TEST_CATALOG, + sdkMajor: 1, + ...context, + }) +} describe('compile scaling', () => { it('finds the cycles of a block wired to many targets in linear time', () => { @@ -14,4 +51,20 @@ describe('compile scaling', () => { return () => stronglyConnected(keys, (key) => (key === 'hub' ? [...targets] : [])) }, 5_000) }) + + it('computes no former-block hint for a reference past the caps', () => { + // Many references per block, so the hint's scan of every block outweighs compiling them. + const knownKeys = new Set(['start', 'gone']) + expectLinearGrowth( + (n) => compileReading(n, n * 80, () => '$ref.gone.x', { knownKeys }), + Math.floor((SOURCE_LIMITS.maxBlocks - 40) / 8) + ) + }) + + it('computes no did-you-mean for an env name past the caps', () => { + expectLinearGrowth((n) => { + const envNames = new Set(Array.from({ length: n }, (_, i) => `KNOWN_${i}`)) + return compileReading(n, n, (i) => `$env.MISSING_${i}`, { envNames }) + }, 200) + }) }) diff --git a/packages/workflow-code/src/compile/values.ts b/packages/workflow-code/src/compile/values.ts index 86c06ecc451..28e197f9846 100644 --- a/packages/workflow-code/src/compile/values.ts +++ b/packages/workflow-code/src/compile/values.ts @@ -57,6 +57,7 @@ import { containerKindOf, enclosingContainers, type JsonFieldEntry, + type LiteralTag, lookup, rangeOf, reportNotImported, @@ -247,9 +248,11 @@ function checkIterationTag( } /** - * `W_REF_NOT_NORMALIZED` for each hand-written `` in literal text whose head is not a - * normalized block key, or names no declared block. Such text is kept verbatim, so a rename in the - * same apply never rewrites it; the block's ident is the reference that follows the block. + * Checks each hand-written `` in literal text whose head is not a normalized block key, + * or names no declared block. Such text is kept verbatim, so a rename in the same apply never + * rewrites it; the block's ident is the reference that follows the block. A head written other + * than as its block's key is `W_REF_NOT_NORMALIZED`. A path whose head names no block at all + * waits for {@link checkUnresolvedTags}, which knows the apply's renames. */ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: ValueSite): void { // A note never runs, so nothing in it resolves: `` in its markdown is text, not a reference. @@ -267,18 +270,59 @@ function checkLiteralRefs(ctx: CompileContext, node: Node, text: string, site: V checkIterationTag(ctx, rangeOf(ctx, node), site.decl.key, segments) continue } - const target = ctx.declsByKey.get(normalized) + const tag = { + token: token.value, + segments, + normalized, + at: rangeOf(ctx, node), + blockKey: site.decl.key, + } + if (!ctx.declsByKey.has(normalized) && segments.length > 1) ctx.unresolvedTags.push(tag) + else reportNotNormalized(ctx, tag) + } +} + +function reportNotNormalized(ctx: CompileContext, tag: LiteralTag): void { + const target = ctx.declsByKey.get(tag.normalized) + ctx.reporter.report( + 'W_REF_NOT_NORMALIZED', + target + ? `${tag.token} is not written with the block key ${tag.normalized}; the text is kept as written.` + : `${tag.token} names no block; the text is kept as written.`, + tag.at, + { + hint: () => + target?.ident + ? `Write \${${formatMemberPath(target.ident, tag.segments.slice(1))}} in a template, so a rename follows the block.` + : 'Reference a declared block through its ident, so a rename follows the block.', + blockKey: tag.blockKey, + } + ) +} + +/** + * Reports each literal `` whose head names no block once the apply's renames are known + * (`renamed`, stored key → source key). The apply rewrites a renamed block's stored key to its new + * one, so such a head still resolves and stays `W_REF_NOT_NORMALIZED`; any other names no block, + * which the run cannot resolve: `W_UNRESOLVED_REF`, refused on a block the edit changed, as the + * server's lint refuses a dangling reference. + */ +export function checkUnresolvedTags( + ctx: CompileContext, + renamed: ReadonlyMap +): void { + for (const tag of ctx.unresolvedTags) { + if (renamed.has(tag.normalized)) { + reportNotNormalized(ctx, tag) + continue + } ctx.reporter.report( - 'W_REF_NOT_NORMALIZED', - normalized === head.name - ? `${token.value} names no block; the text is kept as written.` - : `${token.value} is not written with the block key ${normalized}; the text is kept as written.`, - rangeOf(ctx, node), + 'W_UNRESOLVED_REF', + `${tag.token} names no block, so it does not resolve at run time.`, + tag.at, { - hint: target?.ident - ? `Write \${${formatMemberPath(target.ident, segments.slice(1))}} in a template, so a rename follows the block.` - : 'Reference a declared block through its ident, so a rename follows the block.', - blockKey: site.decl.key, + hint: 'Reference a declared block through its ident, so a rename follows the block.', + blockKey: tag.blockKey, } ) } @@ -611,7 +655,7 @@ function compileRef(ctx: CompileContext, node: Node, site: ValueSite): string | `No workflow variable is keyed ${name}.`, rangeOf(ctx, first), { - hint: didYouMean(key, ctx.variables.keys(), (known) => `vars.${known}`), + hint: () => didYouMean(key, ctx.variables.keys(), (known) => `vars.${known}`), blockKey: site.decl.key, } ) @@ -732,17 +776,20 @@ function segmentsOf( */ function checkEnvName(ctx: CompileContext, name: string, at: Range, blockKey: BlockKey): void { ctx.envReferences.add(name) - if (!ctx.envNames || ctx.envNames.has(name)) return - const near = didYouMean(name, ctx.envNames) + const envNames = ctx.envNames + if (!envNames || envNames.has(name)) return ctx.reporter.report( 'W_ENV_UNKNOWN', `The workspace has no env variable ${name}, so it resolves to nothing at run time.`, at, { - hint: ctx.envDeclaredIn - ? `${near ? `${near} Otherwise d` : 'D'}eclare ${name} in ${ctx.envDeclaredIn} and set its value with \`sim secrets set ${name} --scope workspace\`.` - : (near ?? - `Add ${name} under the workspace's or your environment variables, or use a name .sim/sdk/workspace.d.ts lists.`), + hint: () => { + const near = didYouMean(name, envNames) + return ctx.envDeclaredIn + ? `${near ? `${near} Otherwise d` : 'D'}eclare ${name} in ${ctx.envDeclaredIn} and set its value with \`sim secrets set ${name} --scope workspace\`.` + : (near ?? + `Add ${name} under the workspace's or your environment variables, or use a name .sim/sdk/workspace.d.ts lists.`) + }, blockKey, } ) @@ -997,7 +1044,7 @@ export function checkCodeFileRefs( hint: suggested ? `${ref.tag ? 'A tag' : REF_ROOT} names a block by its key: write ${rewritten(suggested.key)}.` : ctx.knownKeys.has(head.name) - ? formerBlockHint(ctx, head.name, renamed, rewritten, spell) + ? () => formerBlockHint(ctx, head.name, renamed, rewritten, spell) : () => didYouMean(head.name, ctx.declsByKey.keys(), (key) => spell([{ kind: 'name', name: key }]) diff --git a/packages/workflow-code/src/diagnostics/codes.ts b/packages/workflow-code/src/diagnostics/codes.ts index de3caaa0826..4b2b6574609 100644 --- a/packages/workflow-code/src/diagnostics/codes.ts +++ b/packages/workflow-code/src/diagnostics/codes.ts @@ -210,6 +210,18 @@ const COMPILE_CATALOG_CODES = [ 'W_UNKNOWN_MODEL', /** A block of a legacy or deprecated type; `hint` names the type that replaces it. */ 'W_LEGACY_BLOCK_TYPE', + /** + * A new `block()` of a type the catalog does not hold: the server refuses to place it + * (`E_APPLY_INVALID_BLOCK_TYPE`). A block the stored workflow or the lock already holds keeps + * its type quietly, so a workflow saved with a retired type still round-trips. + */ + 'W_UNKNOWN_BLOCK_TYPE', + /** + * A new block of a type the catalog keeps only for blocks already using it (`addable: false`): + * the server refuses to place it (`E_APPLY_TYPE_NOT_ADDABLE`). A block the stored workflow or + * the lock may continue stays quiet, as for `W_UNKNOWN_BLOCK_TYPE`. + */ + 'W_TYPE_NOT_ADDABLE', /** * A schedule created from source with no `timezone`: the editor seeds the viewer's zone, but * a schedule with none runs in UTC. diff --git a/packages/workflow-code/src/diagnostics/refusals.test.ts b/packages/workflow-code/src/diagnostics/refusals.test.ts index e5f0d2f95bb..0c9a3b25cfe 100644 --- a/packages/workflow-code/src/diagnostics/refusals.test.ts +++ b/packages/workflow-code/src/diagnostics/refusals.test.ts @@ -5,15 +5,21 @@ import { FINDING_LIMITS } from './bound' import { MAX_ERRORS } from './codes' import { type EditedBlocks, refuseEditedFindings } from './refusals' -const IMPORTS = "import { api, codeFile, fn, loop, t, trigger, workflow } from '@sim/workflow-sdk'" +const IMPORTS = + "import { api, block, codeFile, fn, loop, t, trigger, workflow } from '@sim/workflow-sdk'" -function compile(body: string[], files: Record = {}) { +function compile( + body: string[], + files: Record = {}, + options: Partial[0]> = {} +) { const text = `// @sim-sdk 1\n${IMPORTS}\n\nexport default workflow('Test', () => {\n${body.map((line) => ` ${line}`).join('\n')}\n})\n` return compileWorkflow({ entry: { path: 'workflow.ts', text }, files: new Map(Object.entries(files)), catalog: TEST_CATALOG, sdkMajor: 1, + ...options, }) } @@ -83,6 +89,94 @@ describe('refuseEditedFindings', () => { expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) }) + it('refuses a literal tag naming no block on a changed block, as the server lint does', () => { + const compiled = compile([ + START, + "const call = api('Call', { url: 'https://x/' })", + "const cased = api('Cased', { url: 'https://x/' })", + "const bare = api('Bare', { url: 'https://x/' })", + 'start.to(call)', + ]) + + expect(errors(refuseEditedFindings(compiled, edited('call', 'cased', 'bare')))).toEqual([ + 'E_APPLY_UNRESOLVED_REF call', + ]) + expect(errors(refuseEditedFindings(compiled, edited('cased', 'bare')))).toEqual([]) + }) + + it('keeps a literal tag naming a block the source renamed, which the apply rewrites', () => { + const compiled = compile( + [START, "const call = api('Call', { url: 'https://x/' })"], + {}, + { renamedKeys: new Map([['former', 'call']]) } + ) + + expect(errors(refuseEditedFindings(compiled, edited('call')))).toEqual([]) + }) + + it('refuses a new block of a type the catalog does not hold, as the placement gate does', () => { + for (const type of ['slak', 'toString', '__proto__']) { + const compiled = compile([START, `const a = block('${type}', 'A', { text: 'hi' })`]) + expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([ + 'E_APPLY_INVALID_BLOCK_TYPE a', + ]) + } + }) + + it('keeps a block of an uncatalogued type the lock already binds', () => { + const compiled = compile( + [START, "const a = block('retired_kind', 'A', { text: 'hi' })"], + {}, + { + storedUnread: { + version: 1, + blocks: { a: { id: 'b-a', parent: null, ident: 'a', order: 0 } }, + variables: {}, + }, + } + ) + expect(compiled.diagnostics.map((d) => d.code)).toEqual([]) + expect(errors(refuseEditedFindings(compiled, edited('a')))).toEqual([]) + }) + + it('refuses a new block of a type the catalog keeps only for blocks already using it', () => { + const catalog = { + ...TEST_CATALOG, + blocks: { + ...TEST_CATALOG.blocks, + api: { ...TEST_CATALOG.blocks.api, addable: false as const }, + }, + } + const body = [ + START, + "const a = api('A', { url: 'x' })", + "const b = block('api', 'B', { url: 'x' })", + ] + + const added = compile(body, {}, { catalog }) + expect(errors(refuseEditedFindings(added, edited('a', 'b')))).toEqual([ + 'E_APPLY_TYPE_NOT_ADDABLE a', + 'E_APPLY_TYPE_NOT_ADDABLE b', + ]) + + const bound = compile( + body, + {}, + { + catalog, + storedUnread: { + version: 1, + blocks: { + a: { id: 'b-a', parent: null, ident: 'a', order: 0 }, + b: { id: 'b-b', parent: null, ident: 'b', order: 1 }, + }, + variables: {}, + }, + } + ) + expect(errors(refuseEditedFindings(bound, edited('a', 'b')))).toEqual([]) + }) + it('refuses an iteration field a shared code file reads for a block outside the loop', () => { const compiled = compile( [ diff --git a/packages/workflow-code/src/diagnostics/refusals.ts b/packages/workflow-code/src/diagnostics/refusals.ts index 6eb96842440..3fde13e735a 100644 --- a/packages/workflow-code/src/diagnostics/refusals.ts +++ b/packages/workflow-code/src/diagnostics/refusals.ts @@ -66,6 +66,8 @@ export type EditRefusalCode = | 'E_APPLY_REF_OUT_OF_SCOPE' | 'E_APPLY_CODE_FILE_REF_SYNTAX' | 'E_APPLY_INVALID_SUBBLOCK_FIELD' + | 'E_APPLY_INVALID_BLOCK_TYPE' + | 'E_APPLY_TYPE_NOT_ADDABLE' /** What an edit changed, as the caller can tell it. */ export interface EditedBlocks { @@ -105,9 +107,17 @@ const PARAM_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OPTION', 'E_APPLY_INVALID_SUBBLOCK_FIELD'], ]) -/** In `workflow.ts` on a block the edit changed: a path no output of the block has. */ +/** + * In `workflow.ts` on a block the edit changed: a path no output of the block has, or a literal + * `` whose head names no block, which the server's lint refuses as a dangling reference, + * or a new block of a type the catalog does not hold or keeps only for blocks already using it, + * which its placement gate refuses. + */ const ENTRY_REFUSALS: ReadonlyMap = new Map([ ['W_UNKNOWN_OUTPUT_PATH', 'E_APPLY_UNRESOLVED_REF'], + ['W_UNRESOLVED_REF', 'E_APPLY_UNRESOLVED_REF'], + ['W_UNKNOWN_BLOCK_TYPE', 'E_APPLY_INVALID_BLOCK_TYPE'], + ['W_TYPE_NOT_ADDABLE', 'E_APPLY_TYPE_NOT_ADDABLE'], ]) /** @@ -132,6 +142,8 @@ export const REFUSABLE_WARNING_CODES: ReadonlySet = new S 'W_UNKNOWN_OUTPUT_PATH', 'W_UNRESOLVED_REF', 'W_CODE_FILE_REF_SYNTAX', + 'W_UNKNOWN_BLOCK_TYPE', + 'W_TYPE_NOT_ADDABLE', ]) /** The rules' error codes with the warning each refuses, for help text. */ diff --git a/packages/workflow-code/src/limits.ts b/packages/workflow-code/src/limits.ts index dd55dc55e5a..f3a8033d215 100644 --- a/packages/workflow-code/src/limits.ts +++ b/packages/workflow-code/src/limits.ts @@ -19,6 +19,8 @@ export const SOURCE_LIMITS = { maxBlocks: 2_000, /** Edges in one workflow. Mirrors the v2 graph cap (`MAX_WORKFLOW_GRAPH_EDGES`). */ maxEdges: 10_000, + /** Variables in one workflow, so a lock holds at most this many variable entries. */ + maxVariables: 2_000, /** Files in one source: `workflow.ts` plus one code file per block. */ maxFiles: 2_001, /** UTF-8 bytes of every file together, the entry included. */ @@ -34,7 +36,7 @@ export const SOURCE_LIMITS = { /** * Body cap for a request carrying source files: every file at its limit, doubled for JSON string * escaping (quotes, backslashes, newlines and tabs each escape to two bytes), plus room for a lock - * of `maxBlocks` entries. + * of `maxBlocks` block and `maxVariables` variable entries. */ export const SOURCE_REQUEST_MAX_BYTES = 2 * SOURCE_LIMITS.maxTotalBytes + 2_000_000 @@ -78,7 +80,7 @@ export function isCodeFilePath(path: string): boolean { /** One size limit a source exceeds, with the measured value. */ export interface SourceLimitExcess { - limit: 'maxBlocks' | 'maxEdges' | 'maxFiles' | 'maxTotalBytes' + limit: 'maxBlocks' | 'maxEdges' | 'maxVariables' | 'maxFiles' | 'maxTotalBytes' actual: number max: number } @@ -103,12 +105,12 @@ export function utf8Bytes(text: string, cap = Number.POSITIVE_INFINITY): number } /** - * Every size limit a printed source of a workflow with `counts` blocks and edges exceeds, in limit - * order; empty when it fits. Depth is `sourceDepthOverflow`'s. + * Every size limit a printed source of a workflow with `counts` blocks, edges and variables + * exceeds, in limit order; empty when it fits. Depth is `sourceDepthOverflow`'s. */ export function sourceLimitExcesses( files: Readonly>, - counts: { blocks: number; edges: number } + counts: { blocks: number; edges: number; variables: number } ): SourceLimitExcess[] { const excesses: SourceLimitExcess[] = [] if (counts.blocks > SOURCE_LIMITS.maxBlocks) { @@ -117,6 +119,13 @@ export function sourceLimitExcesses( if (counts.edges > SOURCE_LIMITS.maxEdges) { excesses.push({ limit: 'maxEdges', actual: counts.edges, max: SOURCE_LIMITS.maxEdges }) } + if (counts.variables > SOURCE_LIMITS.maxVariables) { + excesses.push({ + limit: 'maxVariables', + actual: counts.variables, + max: SOURCE_LIMITS.maxVariables, + }) + } const paths = Object.keys(files) if (paths.length > SOURCE_LIMITS.maxFiles) { excesses.push({ limit: 'maxFiles', actual: paths.length, max: SOURCE_LIMITS.maxFiles }) diff --git a/packages/workflow-code/src/lock.test.ts b/packages/workflow-code/src/lock.test.ts index 54ba0becbb5..51b4241f3a1 100644 --- a/packages/workflow-code/src/lock.test.ts +++ b/packages/workflow-code/src/lock.test.ts @@ -1,4 +1,6 @@ +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' import { describe, expect, it } from 'vitest' +import { SOURCE_LIMITS } from './limits' import { parseSourceBase, parseWorkflowLock } from './lock' function validLock(): Record { @@ -175,6 +177,51 @@ describe('parseWorkflowLock', () => { ]) }) + it('reports only the blocks on a cycle, not the blocks that lead into one', () => { + const lock = validLock() + lock.blocks = { + a: { id: 'a', parent: 'b', ident: 'a', order: 0 }, + b: { id: 'b', parent: 'a', ident: 'b', order: 1 }, + c: { id: 'c', parent: 'a', ident: 'c', order: 2 }, + d: { id: 'd', parent: 'c', ident: 'd', order: 3 }, + } + expect(codesFor(lock)).toEqual([ + 'E_LOCK_PARENT /blocks/a/parent', + 'E_LOCK_PARENT /blocks/b/parent', + ]) + }) + + it('checks a long parent chain in linear time', () => { + // 64 to 512 blocks, not the cap: a chain at the cap outgrows the CPU cache, which narrows the + // gap between linear and quadratic growth. + expectLinearGrowth((n) => { + const blocks: Record = {} + for (let i = 0; i < n; i += 1) { + blocks[`b${i}`] = { + id: `id${i}`, + parent: i === 0 ? null : `b${i - 1}`, + ident: `b${i}`, + order: i, + } + } + const lock = { version: 1, blocks, variables: {} } + return () => parseWorkflowLock(lock) + }, 64) + }) + + it('refuses more blocks or variables than a workflow may hold, before reading them', () => { + const blocks: Record = {} + const variables: Record = {} + // Each entry is malformed, so a reader that walked them would report far more than the cap. + for (let i = 0; i <= SOURCE_LIMITS.maxBlocks; i += 1) blocks[`b${i}`] = { id: i } + for (let i = 0; i <= SOURCE_LIMITS.maxVariables; i += 1) variables[`v${i}`] = '' + expect(codesFor({ version: 1, blocks, variables: {} })).toEqual(['E_LOCK_SHAPE /blocks']) + expect(codesFor({ version: 1, blocks: {}, variables })).toEqual(['E_LOCK_SHAPE /variables']) + // An oversized variables record is refused without walking the malformed blocks beside it. + const atCap = Object.fromEntries(Object.entries(blocks).slice(0, SOURCE_LIMITS.maxBlocks)) + expect(codesFor({ version: 1, blocks: atCap, variables })).toEqual(['E_LOCK_SHAPE /variables']) + }) + it('rejects branch keys that are not identifiers', () => { expect(codesFor(withBlock('route', { branches: { 'else-if-1': 'x' } }))).toEqual([ 'E_LOCK_BRANCH /blocks/route/branches/else-if-1', diff --git a/packages/workflow-code/src/lock.ts b/packages/workflow-code/src/lock.ts index 52b7d0ba015..5b868ccad0d 100644 --- a/packages/workflow-code/src/lock.ts +++ b/packages/workflow-code/src/lock.ts @@ -22,6 +22,7 @@ import { isIdentifierName } from './code-files/ref-path' import type { LockDiagnosticCode } from './diagnostics/codes' import { diagnostic } from './diagnostics/factory' import type { Diagnostic } from './diagnostics/frame' +import { SOURCE_LIMITS } from './limits' import type { BlockKey, BranchKey, Json, SubBlockId, VarKey } from './model' import { SDK_RESERVED_IDENTIFIERS } from './sdk' @@ -474,6 +475,7 @@ class LockReader { } } + const cyclic = blocksOnParentCycles(blocks) for (const [key, block] of Object.entries(blocks)) { if (block.parent === null) continue if (!Object.hasOwn(blocks, block.parent)) { @@ -482,21 +484,19 @@ class LockReader { ['blocks', key, 'parent'], `Parent "${block.parent}" is not a block in this lock.` ) - continue - } - const visited = new Set() - let current: BlockKey | null = block.parent - while (current !== null && Object.hasOwn(blocks, current) && !visited.has(current)) { - if (current === key) { - this.report('E_LOCK_PARENT', ['blocks', key, 'parent'], `"${key}" contains itself.`) - break - } - visited.add(current) - current = blocks[current].parent + } else if (cyclic.has(key)) { + this.report('E_LOCK_PARENT', ['blocks', key, 'parent'], `"${key}" contains itself.`) } } } + /** Whether `raw` holds more than `max` entries, a workflow's cap; reported so it is never read. */ + overCap(raw: unknown, max: number, at: readonly string[]): boolean { + if (!isRecordLike(raw) || Object.keys(raw).length <= max) return false + this.report('E_LOCK_SHAPE', at, `Expected at most ${max} entries.`) + return true + } + variables(raw: unknown): Record { const variables = this.idRecord(raw, ['variables']) for (const key of Object.keys(variables)) { @@ -512,6 +512,33 @@ class LockReader { } } +/** + * The blocks whose parent chain leads back to themselves. Each block is walked once: a walk stops + * at a block an earlier walk finished, and the blocks from where it meets its own path onward are + * the cycle. + */ +function blocksOnParentCycles(blocks: Record): Set { + const cyclic = new Set() + const finished = new Set() + for (const start of Object.keys(blocks)) { + const path: BlockKey[] = [] + const onPath = new Map() + let current: BlockKey | null = start + while (current !== null && Object.hasOwn(blocks, current) && !finished.has(current)) { + const index = onPath.get(current) + if (index !== undefined) { + for (const key of path.slice(index)) cyclic.add(key) + break + } + onPath.set(current, path.length) + path.push(current) + current = blocks[current].parent + } + for (const key of path) finished.add(key) + } + return cyclic +} + /** * A lock object of any version this build reads, as the current version (1) holds it, or `null` * for a version it cannot read. The one seam a lock format change goes through: v1 is current, @@ -602,15 +629,20 @@ export function parseWorkflowLock(input: unknown): ParseWorkflowLockResult { return { diagnostics: reader.diagnostics } } + // Both counts come before either record is read, so an oversized one never costs a walk of the other. + const blocksOverCap = reader.overCap(json.blocks, SOURCE_LIMITS.maxBlocks, ['blocks']) + const variablesOverCap = reader.overCap(json.variables, SOURCE_LIMITS.maxVariables, ['variables']) + if (blocksOverCap || variablesOverCap) return { diagnostics: reader.diagnostics } + const extensions = reader.extensions(json, LOCK_FIELDS, []) const blockEntries: Array<[BlockKey, WorkflowLockBlock]> = [] - if (isRecordLike(json.blocks)) { + if (!isRecordLike(json.blocks)) { + reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') + } else { for (const [key, raw] of Object.entries(json.blocks)) { const block = reader.block(key, raw) if (block) blockEntries.push([key, block]) } - } else { - reader.report('E_LOCK_SHAPE', ['blocks'], '"blocks" must be an object.') } const blocks: Record = Object.fromEntries(blockEntries) reader.crossCheckBlocks(blocks) @@ -634,8 +666,10 @@ export interface SourceHints { /** * Lock keys the source renamed, old → new: a source block under a key `isStored` rejects whose * ident is that of exactly one lock entry, a stored block's, whose key is gone from the source. - * This is materialize's `lock-ident` identity rule, so a compile that pairs source blocks with - * stored ones by key agrees with materialize on which stored block a renamed one continues. + * This is materialize's `lock-ident` identity rule for a dropped key, so a compile that pairs + * source blocks with stored ones by key agrees with materialize on which stored block a renamed + * one continues. A swap where both keys survive, which materialize also binds by ident (DESIGN.md + * rule 2), is left out: the compile reads it by key. */ export function lockIdentRenames( hints: SourceHints, diff --git a/packages/workflow-code/src/materialize/identity.ts b/packages/workflow-code/src/materialize/identity.ts index 7d325c35052..12558890949 100644 --- a/packages/workflow-code/src/materialize/identity.ts +++ b/packages/workflow-code/src/materialize/identity.ts @@ -15,8 +15,9 @@ import { isSimNode, type VarKey, } from '../model' -import { deepEqual, getOwn } from '../records' +import { deepEqual, getOwn, pushTo, strings } from '../records' import type { MaterializeHooks } from './context' +import { settingsIndex } from './settings-index' /** How identity rule matched a compiled block to a stored one (rules 1–4). */ type IdentityMatchRule = 'lock-key' | 'lock-ident' | 'db-name' | 'inferred' @@ -156,11 +157,11 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { inputs.compiledIdents ? getOwn(inputs.compiledIdents, key) : undefined const resolved = new Map() - /** The ids `resolved` holds, so a lock id's reuse check is one lookup. */ - const resolvedIds = new Set() + /** The key `resolved` holds each id under, so finding an id's block is one lookup. */ + const keyOfId = new Map() const resolve = (key: BlockKey, block: ResolvedBlock) => { resolved.set(key, block) - resolvedIds.add(block.id) + keyOfId.set(block.id, key) } const claimed = new Set() /** Keys that end as created blocks no later rule may match (retype, or a lock id to reuse). */ @@ -258,6 +259,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { workflow, prev, resolved, + keyOfId, claimed, open, match, @@ -278,7 +280,7 @@ export function resolveIdentity(inputs: IdentityInputs): IdentityResolution { lockId !== undefined && !claimed.has(lockId) && !Object.hasOwn(prevBlocks, lockId) && - !resolvedIds.has(lockId) + !keyOfId.has(lockId) if (lockId !== undefined && !reusable) { diagnostics.push( diagnostic( @@ -341,6 +343,7 @@ interface InferenceState { workflow: CanonicalWorkflow prev: PrevGraph resolved: Map + keyOfId: ReadonlyMap claimed: Set open: () => BlockKey[] match: (key: BlockKey, prevId: string, via: IdentityMatchRule) => boolean @@ -354,14 +357,26 @@ interface InferenceState { catalog: BlockCatalog } +type CanonicalEdges = CanonicalWorkflow['edges'] + +/** Each block's edges, in or out, so a block's neighbourhood reads only its own edges. */ +function edgesByEndpoint(edges: CanonicalEdges): Map { + const byKey = new Map() + for (const edge of edges) { + pushTo(byKey, edge.from, edge) + if (edge.to !== edge.from) pushTo(byKey, edge.to, edge) + } + return byKey +} + /** `in:` / `out:` for every already-bound neighbour of a block. */ function neighbourhood( - edges: CanonicalWorkflow['edges'], + edgesAt: ReadonlyMap, key: BlockKey, idOf: (key: BlockKey) => string | undefined ): Set { const result = new Set() - for (const edge of edges) { + for (const edge of edgesAt.get(key) ?? []) { if (edge.from === key) { const id = idOf(edge.to) if (id !== undefined) result.add(`out:${id}`) @@ -378,17 +393,16 @@ function sameSet(left: Set, right: Set): boolean { return left.size === right.size && [...left].every((item) => right.has(item)) } -/** A block's params with every `` reference token rewritten to `to`. */ -function withKeyRenamed(params: CanonicalBlock['params'], from: BlockKey, to: BlockKey) { - const rewrite = (value: unknown): unknown => { - if (typeof value === 'string') { - return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) - } - if (Array.isArray(value)) return value.map(rewrite) - if (value === null || typeof value !== 'object') return value - return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewrite(item)])) +/** `value` with every `` reference token rewritten to `to`. */ +function withKeyRenamed(value: unknown, from: BlockKey, to: BlockKey): unknown { + if (typeof value === 'string') { + return value.split(`<${from}.`).join(`<${to}.`).split(`<${from}>`).join(`<${to}>`) } - return rewrite(params) + if (Array.isArray(value)) return value.map((item) => withKeyRenamed(item, from, to)) + if (value === null || typeof value !== 'object') return value + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, withKeyRenamed(item, from, to)]) + ) } /** @@ -402,35 +416,49 @@ function settingsAgree( key: BlockKey, prevKey: BlockKey ): boolean { - const left = target.params - const right = withKeyRenamed(prev.params, prevKey, key) as CanonicalBlock['params'] - const shared = Object.keys(left).filter((id) => Object.hasOwn(right, id)) - return shared.length > 0 && shared.every((id) => deepEqual(left[id], right[id])) -} - -/** Every string under `value`, depth first. */ -function* strings(value: unknown): Generator { - if (typeof value === 'string') yield value - else if (Array.isArray(value)) for (const item of value) yield* strings(item) - else if (value !== null && typeof value === 'object') { - for (const item of Object.values(value)) yield* strings(item) + let shared = false + for (const [id, value] of Object.entries(target.params)) { + if (!Object.hasOwn(prev.params, id)) continue + if (!deepEqual(value, withKeyRenamed(prev.params[id], prevKey, key))) return false + shared = true } + return shared } /** - * Whether a block holds a reference to `key` in any field a reference can sit in (its params, a - * condition or router's branches, a loop or parallel's collection), its head read as the resolver - * reads it: normalized, so `` references `fetch`. + * The keys a block references in any field a reference can sit in (its params, a condition or + * router's branches, a loop or parallel's collection), each head read as the resolver reads it: + * normalized, so `` references `fetch`. */ -function referencesKey(block: CanonicalBlock, key: BlockKey): boolean { +function referencedKeys(block: CanonicalBlock): Set { + const keys = new Set() for (const text of strings([block.params, block.branches, block.subflow])) { for (const token of findWorkflowReferenceTokens(text)) { if (token.kind !== 'workflow') continue const head = parseStoredPath(token.value.slice(1, -1))?.[0] - if (head?.kind === 'name' && normalizeWorkflowBlockName(head.name) === key) return true + if (head?.kind === 'name') keys.add(normalizeWorkflowBlockName(head.name)) } } - return false + return keys +} + +/** + * The first half (rounded up) of a block's bound neighbours in `order`. Two blocks share a + * majority of their neighbours, the `connections` signal, only if these halves meet: the earliest + * shared neighbour sits in both, since more than half of each side's neighbours are shared and + * come at or after it. + */ +function neighbourPrefix(items: Set, order: (left: string, right: string) => number) { + return [...items].sort(order).slice(0, Math.ceil(items.size / 2)) +} + +/** Each key's items, inverted: item → the keys that hold it. */ +function invert(entries: Iterable<[BlockKey, Iterable]>): Map { + const byItem = new Map() + for (const [key, items] of entries) { + for (const item of items) pushTo(byItem, item, key) + } + return byItem } /** @@ -553,10 +581,38 @@ function inferRenames(state: InferenceState): void { return true } - const neighbours = (key: BlockKey, prevKey: BlockKey) => ({ - target: neighbourhood(workflow.edges, key, idOfTarget), - prev: neighbourhood(prevCanonical.edges, prevKey, idOfPrev), - }) + const targetEdges = edgesByEndpoint(workflow.edges) + const prevEdges = edgesByEndpoint(prevCanonical.edges) + const targetSide = (key: BlockKey) => neighbourhood(targetEdges, key, idOfTarget) + const prevSide = (prevKey: BlockKey) => neighbourhood(prevEdges, prevKey, idOfPrev) + + /** Each stored block's referrers: the other stored blocks that reference its key. */ + const referrersOf = new Map() + for (const [other, block] of Object.entries(prevCanonical.blocks)) { + for (const key of referencedKeys(block)) { + if (key !== other && Object.hasOwn(prevCanonical.blocks, key)) pushTo(referrersOf, key, other) + } + } + const targetReferences = new Map>() + const referencedByTarget = (key: BlockKey): Set => { + let keys = targetReferences.get(key) + if (!keys) { + keys = referencedKeys(workflow.blocks[key]) + targetReferences.set(key, keys) + } + return keys + } + /** The blocks that now stand where `prevKey`'s referrers stood, or none when one is unbound. */ + const renamedReferrers = (prevKey: BlockKey): BlockKey[] => { + const now: BlockKey[] = [] + for (const other of referrersOf.get(prevKey) ?? []) { + const id = idOfPrev(other) + const key = id === undefined ? undefined : state.keyOfId.get(id) + if (key === undefined) return [] + now.push(key) + } + return now + } const byConnections = (): boolean => { let progressed = false @@ -564,8 +620,7 @@ function inferRenames(state: InferenceState): void { if (targets.length !== 1 || prevKeys.length !== 1) continue const [key] = targets const [prevKey] = prevKeys - const sides = neighbours(key, prevKey) - if (!sameSet(sides.target, sides.prev)) continue + if (!sameSet(targetSide(key), prevSide(prevKey))) continue const triggerHeld = isTriggerLike(workflow.blocks[key], catalog) && !settingsAgree(workflow.blocks[key], prevCanonical.blocks[prevKey], key, prevKey) @@ -574,47 +629,127 @@ function inferRenames(state: InferenceState): void { return progressed } + /** Bound neighbourhoods of one group's blocks, read once per group. */ + interface GroupSides { + targets: Map> + vacated: Map> + } + /** The strong signal tying new block `key` to vacated stored block `prevKey`, if any. */ - const signal = (key: BlockKey, prevKey: BlockKey): InferenceBasis | null => { + const signal = (key: BlockKey, prevKey: BlockKey, sides: GroupSides): InferenceBasis | null => { const target = workflow.blocks[key] if (settingsAgree(target, prevCanonical.blocks[prevKey], key, prevKey)) return 'settings' if (isTriggerLike(target, catalog)) return null - const sides = neighbours(key, prevKey) - const shared = [...sides.target].filter((item) => sides.prev.has(item)).length - const union = new Set([...sides.target, ...sides.prev]).size + const targetItems = sides.targets.get(key) ?? new Set() + const prevItems = sides.vacated.get(prevKey) ?? new Set() + const shared = [...targetItems].filter((item) => prevItems.has(item)).length + const union = targetItems.size + prevItems.size - shared if (shared > 0 && shared * 2 > union) return 'connections' - const referrers = Object.keys(prevCanonical.blocks).filter( - (other) => other !== prevKey && referencesKey(prevCanonical.blocks[other], prevKey) - ) + const referrers = renamedReferrers(prevKey) const renamedAtEverySite = - referrers.length > 0 && - referrers.every((other) => { - const id = idOfPrev(other) - const now = id === undefined ? undefined : [...resolved].find(([, b]) => b.id === id)?.[0] - return now !== undefined && referencesKey(workflow.blocks[now], key) - }) + referrers.length > 0 && referrers.every((now) => referencedByTarget(now).has(key)) return renamedAtEverySite ? 'references' : null } + /** + * The pairs of one group that may show a signal, found from either side without judging every + * pair: equal settings ({@link settingsIndex}), shared neighbours ({@link neighbourPrefix}), or + * every referrer now referencing the new block. A superset, so {@link signal} need judge only + * these. + */ + const groupSearch = (sides: GroupSides) => { + const blocksOf = (keys: Iterable, from: CanonicalWorkflow) => + [...keys].map((key): [BlockKey, CanonicalBlock] => [key, from.blocks[key]]) + const vacatedBySettings = settingsIndex(blocksOf(sides.vacated.keys(), prevCanonical)) + const targetsBySettings = settingsIndex(blocksOf(sides.targets.keys(), workflow)) + + const frequency = new Map() + for (const items of [...sides.targets.values(), ...sides.vacated.values()]) { + for (const item of items) frequency.set(item, (frequency.get(item) ?? 0) + 1) + } + const order = (left: string, right: string) => + (frequency.get(left) ?? 0) - (frequency.get(right) ?? 0) || (left < right ? -1 : 1) + const prefixes = (side: Map>) => + new Map([...side].map(([key, items]) => [key, neighbourPrefix(items, order)])) + const targetPrefixes = prefixes(sides.targets) + const vacatedPrefixes = prefixes(sides.vacated) + const vacatedByNeighbour = invert(vacatedPrefixes) + const targetsByNeighbour = invert(targetPrefixes) + + const referenced = new Map() + for (const prevKey of sides.vacated.keys()) { + const [first, ...rest] = renamedReferrers(prevKey).map(referencedByTarget) + referenced.set( + prevKey, + [...(first ?? [])].filter( + (key) => sides.targets.has(key) && rest.every((keys) => keys.has(key)) + ) + ) + } + const referencing = invert(referenced) + + function* unique(lists: Iterable>): Generator { + const seen = new Set() + for (const list of lists) { + for (const key of list) { + if (seen.has(key)) continue + seen.add(key) + yield key + } + } + } + return { + vacatedFor: (key: BlockKey) => + unique([ + vacatedBySettings(workflow.blocks[key]), + ...(targetPrefixes.get(key) ?? []).map((item) => vacatedByNeighbour.get(item) ?? []), + referencing.get(key) ?? [], + ]), + targetsFor: (prevKey: BlockKey) => + unique([ + targetsBySettings(prevCanonical.blocks[prevKey]), + ...(vacatedPrefixes.get(prevKey) ?? []).map((item) => targetsByNeighbour.get(item) ?? []), + referenced.get(prevKey) ?? [], + ]), + } + } + + /** + * Pairs each new block that shows a signal for exactly one vacated block, which no other new + * block shows one for. A new block stops at its second signal, since it can no longer pair; a + * vacated block's rivals are then sought only for the new blocks that would pair with it. + */ const byVacatedLock = (): boolean => { let progressed = false for (const { targets, prevKeys } of groups()) { const vacated = prevKeys.filter((prevKey) => lockedIds.has(prevIdentity[prevKey].id)) if (vacated.length === 0) continue - const signals = new Map>() + const sides: GroupSides = { + targets: new Map(targets.map((key) => [key, targetSide(key)])), + vacated: new Map(vacated.map((prevKey) => [prevKey, prevSide(prevKey)])), + } + const search = groupSearch(sides) + const chosen: Array<[BlockKey, BlockKey, InferenceBasis]> = [] for (const key of targets) { - const found = new Map() - for (const prevKey of vacated) { - const basis = signal(key, prevKey) - if (basis) found.set(prevKey, basis) + const found: Array<[BlockKey, InferenceBasis]> = [] + for (const prevKey of search.vacatedFor(key)) { + const basis = signal(key, prevKey, sides) + if (basis) found.push([prevKey, basis]) + if (found.length > 1) break + } + if (found.length !== 1) continue + const [[prevKey, basis]] = found + let rivalled = false + for (const other of search.targetsFor(prevKey)) { + if (other !== key && signal(other, prevKey, sides) !== null) { + rivalled = true + break + } } - signals.set(key, found) + if (!rivalled) chosen.push([key, prevKey, basis]) } - for (const [key, found] of signals) { - if (found.size !== 1) continue - const [[prevKey, basis]] = [...found] - const rivals = [...signals.values()].filter((other) => other.has(prevKey)).length - if (rivals === 1 && pair(key, prevKey, basis)) progressed = true + for (const [key, prevKey, basis] of chosen) { + if (pair(key, prevKey, basis)) progressed = true } } return progressed diff --git a/packages/workflow-code/src/materialize/index.test.ts b/packages/workflow-code/src/materialize/index.test.ts index 519285156ef..9a1b7cb31f2 100644 --- a/packages/workflow-code/src/materialize/index.test.ts +++ b/packages/workflow-code/src/materialize/index.test.ts @@ -676,6 +676,23 @@ describe('materialize: identity', () => { } ) + it('infers a rename by its renamed references beside a reference to no block', () => { + const prev = base() + const lock = lockFor(prev) + const target = edit(renameFetch(canonical(prev, lock).workflow), (draft) => { + draft.blocks.load.params.code = { kind: 'value', value: 'return 2' } + draft.blocks.report.params.code = { + kind: 'value', + value: 'return + ', + } + draft.edges = draft.edges.filter((e) => e.from !== 'start') + }) + const result = run(target, lock, { prev, compiledIdents: { load: 'x_load' } }) + expect(result.entries).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'load', id: 'a1', via: 'inferred' }) + ) + }) + it('infers a rename under a fresh lock when the ident changed too', () => { const prev = base() const lock = lockFor(prev) @@ -958,6 +975,28 @@ describe('materialize: identity', () => { expect(ids(renamed('count', 'number', 'Rows to fetch'))).toEqual(['new-1', 'f-q']) }) + it('keeps the stored variable order when source lists the variables in another order', () => { + const pulled = workflowState([blockState('a1', 'function', 'Fetch', { code: 'return 1' })], { + variables: { + 'var-z': { id: 'var-z', name: 'Zeta', type: 'number', value: 1 }, + 'var-a': { id: 'var-a', name: 'Alpha', type: 'number', value: 1 }, + }, + }) + const edited = edit(canonical(pulled).workflow, (draft) => { + draft.variables = { + alpha: { ...draft.variables.alpha, value: 2 }, + new: { key: 'new', name: 'New', type: 'number', value: 3 }, + zeta: draft.variables.zeta, + } + }) + + expect(Object.keys(run(edited, null, { prev: pulled }).state.variables ?? {})).toEqual([ + 'var-z', + 'var-a', + 'new-1', + ]) + }) + it('keeps the id of the one variable renamed to a key of the same type', () => { const pulled = base() const renamed = (type: 'number' | 'plain') => diff --git a/packages/workflow-code/src/materialize/index.ts b/packages/workflow-code/src/materialize/index.ts index f353768b423..a8afd0ba639 100644 --- a/packages/workflow-code/src/materialize/index.ts +++ b/packages/workflow-code/src/materialize/index.ts @@ -40,7 +40,7 @@ import { rewriteRenamedReferences, rewriteStoredBlockReferences } from './values * Variables keep their ids by key: the lock's id, then prev's, else a new one. When exactly one * variable's key is new and exactly one prev variable's key is gone, with the same type, the * variable is a rename and keeps that id. A variable canonical-equal to the prev variable with - * that id is prev's stored object unchanged. + * that id is prev's stored object unchanged, and continuing variables keep their stored order. */ function writeVariables( workflow: CanonicalWorkflow, @@ -103,7 +103,27 @@ function writeVariables( : { ...stored, id, name: variable.name, type: variable.type, value: variable.value } ) } - return { variables, idByKey } + return { variables: inStoredOrder(variables, prevVariables), idByKey } +} + +/** + * `variables` with the ones `stored` holds first, in its order, then the new ones in source order. + * The panel lists variables in record order and the printer sorts them by key, so writing them in + * source order would reorder the panel on every apply that touches a variable. + */ +function inStoredOrder( + variables: Record, + stored: Readonly> +): Record { + const ordered: Record = {} + for (const id of Object.keys(stored)) { + const variable = getOwn(variables, id) + if (variable) setOwn(ordered, id, variable) + } + for (const [id, variable] of Object.entries(variables)) { + if (!Object.hasOwn(ordered, id)) setOwn(ordered, id, variable) + } + return ordered } /** The one variable a lone key rename left without a known id, with its id (see {@link soleRename}). */ diff --git a/packages/workflow-code/src/materialize/scaling.test.ts b/packages/workflow-code/src/materialize/scaling.test.ts new file mode 100644 index 00000000000..29c728b8f3a --- /dev/null +++ b/packages/workflow-code/src/materialize/scaling.test.ts @@ -0,0 +1,109 @@ +/** + * Rename inference (identity rule 4) grows linearly with the number of renamed blocks (see + * `expectLinearGrowth` for how a case is measured). + */ + +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' +import { describe, expect, it } from 'vitest' +import type { CanonicalBlock } from '../model' +import { + blockState, + CATALOG, + canonical, + edge, + fixtureHooks, + lockFor, + RESOURCES, + workflowState, +} from './fixtures.test-helpers' +import { materialize } from './index' + +interface RenameShape { + /** The readers now read the new blocks, so each pairs by its renamed references. */ + renameReaders?: boolean + /** Every function block hangs off one hub, so every pair shares its one neighbour. */ + hub?: boolean +} + +/** + * `n` function blocks, each read by its own api block, replaced by `n` new function blocks under + * new keys, idents and code. Without a signal ({@link RenameShape}) each new block is created. + */ +function renameAll(n: number, { renameReaders = false, hub = false }: RenameShape) { + const prev = workflowState( + [ + blockState('hub', 'api', 'Hub', { url: 'https://hub' }), + ...Array.from({ length: n }, (_, i) => [ + blockState(`fn${i}`, 'function', `Old${i}`, { + language: 'javascript', + code: `return ${i}`, + }), + blockState(`rd${i}`, 'api', `Reader${i}`, { url: `https://x/` }), + ]).flat(), + ], + { edges: hub ? Array.from({ length: n }, (_, i) => edge(`e${i}`, 'hub', `fn${i}`)) : [] } + ) + const lock = lockFor(prev) + const prevCanonical = canonical(prev, lock) + const blocks: Record = {} + for (const [key, block] of Object.entries(prevCanonical.workflow.blocks)) { + if (block.type === 'function') { + const index = key.slice('old'.length) + const next = `new${index}` + blocks[next] = { + ...block, + key: next, + name: `New${index}`, + params: { ...block.params, code: { kind: 'value', value: `return '${next}'` } }, + } + } else { + const params = JSON.stringify(block.params) + blocks[key] = { + ...block, + params: JSON.parse(renameReaders ? params.replace(/ ({ + ...link, + to: link.to.replace(/^old/, 'new'), + })) + const workflow = { ...prevCanonical.workflow, blocks, edges } + const compiledIdents = Object.fromEntries(Object.keys(blocks).map((key) => [key, `x_${key}`])) + return () => + materialize(workflow, lock, { + catalog: CATALOG, + hooks: fixtureHooks(), + resources: RESOURCES, + prev, + prevCanonical, + lockFreshness: 'fresh', + compiledIdents, + }) +} + +describe('materialize scaling', () => { + it('pairs blocks renamed with their references, and nothing else', () => { + const viaReferences = renameAll(3, { renameReaders: true })().entries + expect(viaReferences.filter((entry) => entry.status === 'created')).toEqual([]) + expect(viaReferences).toContainEqual( + expect.objectContaining({ status: 'matched', key: 'new1', id: 'fn1', via: 'inferred' }) + ) + for (const shape of [{}, { hub: true }]) { + const unpaired = renameAll(3, shape)().entries + expect(unpaired.filter((entry) => entry.status === 'created')).toHaveLength(3) + } + }) + + it('infers renames in linear time when no new block shows a signal', () => { + expectLinearGrowth((n) => renameAll(n, {}), 16) + }) + + it('infers renames in linear time when every new block pairs by its references', () => { + expectLinearGrowth((n) => renameAll(n, { renameReaders: true }), 16) + }) + + it('infers renames in linear time when every pair shares its one neighbour', () => { + expectLinearGrowth((n) => renameAll(n, { hub: true }), 16) + }) +}) diff --git a/packages/workflow-code/src/materialize/settings-index.test.ts b/packages/workflow-code/src/materialize/settings-index.test.ts new file mode 100644 index 00000000000..1ba613339d7 --- /dev/null +++ b/packages/workflow-code/src/materialize/settings-index.test.ts @@ -0,0 +1,33 @@ +import { expectLinearGrowth } from '@sim/testing/helpers/scaling' +import { describe, it } from 'vitest' +import type { CanonicalBlock } from '../model' +import { settingsIndex } from './settings-index' + +/** `n` function blocks sharing a language, each with its own code, so none settles a match. */ +function functionBlocks( + n: number, + prefix: string +): Array<[string, Pick]> { + return Array.from({ length: n }, (_, i) => [ + `${prefix}${i}`, + { + params: { + language: { kind: 'value', value: 'javascript' }, + code: { kind: 'value', value: `return '${prefix}${i}'` }, + }, + }, + ]) +} + +describe('settingsIndex', () => { + it('looks up blocks that all hold the narrowing param, each with its own value, in linear time', () => { + expectLinearGrowth((n) => { + const indexed = functionBlocks(n, 'old') + const lookups = functionBlocks(n, 'new') + return () => { + const matches = settingsIndex(indexed) + for (const [, block] of lookups) Array.from(matches(block)) + } + }, 250) + }) +}) diff --git a/packages/workflow-code/src/materialize/settings-index.ts b/packages/workflow-code/src/materialize/settings-index.ts new file mode 100644 index 00000000000..0589b0a57f5 --- /dev/null +++ b/packages/workflow-code/src/materialize/settings-index.ts @@ -0,0 +1,63 @@ +import { canonicalJson } from '@sim/utils/object' +import type { BlockKey, CanonicalBlock } from '../model' +import { pushTo, strings } from '../records' + +/** Whether any string under `value` holds a `<`, the only way a reference can sit in it. */ +function holdsAngle(value: unknown): boolean { + for (const text of strings(value)) if (text.includes('<')) return true + return false +} + +/** + * The blocks among `blocks` whose settings may agree with a given block's, a superset found + * without comparing every pair, and the same whichever side is indexed. Settings agree only when + * every param both blocks hold is equal (references to a renamed block's old key read as its new + * one), so a match either lacks a param the given block holds or holds it equal. A value with no + * `<` holds no reference, so the old-key rewrite leaves it as it is and equal values have equal + * canonical JSON; a value with one keeps one after the rewrite, so it can only equal another such + * value. + * + * A lookup walks only the param it narrows to: the blocks holding that param's value (its bucket) + * and the blocks lacking the param, each list built once. + */ +export function settingsIndex(blocks: ReadonlyArray<[BlockKey, Pick]>) { + const byValue = new Map() + const holders = new Map>() + const bucketOf = (id: string, value: unknown) => + holdsAngle(value) ? `<\u0000${id}` : `=\u0000${id}\u0000${canonicalJson(value)}` + for (const [key, block] of blocks) { + for (const [id, value] of Object.entries(block.params)) { + pushTo(byValue, bucketOf(id, value), key) + const held = holders.get(id) + if (held) held.add(key) + else holders.set(id, new Set([key])) + } + } + const lacking = new Map() + const lackingOf = (id: string): BlockKey[] => { + let keys = lacking.get(id) + if (!keys) { + const held = holders.get(id) + keys = held + ? blocks.flatMap(([key]) => (held.has(key) ? [] : [key])) + : blocks.map(([key]) => key) + lacking.set(id, keys) + } + return keys + } + return function* matches(block: Pick): Generator { + let best: { id: string; equal: BlockKey[] } | undefined + let bestSize = Number.POSITIVE_INFINITY + for (const [id, value] of Object.entries(block.params)) { + const equal = byValue.get(bucketOf(id, value)) ?? [] + const size = blocks.length - (holders.get(id)?.size ?? 0) + equal.length + if (size < bestSize) { + best = { id, equal } + bestSize = size + } + } + if (!best) return + yield* lackingOf(best.id) + yield* best.equal + } +} diff --git a/packages/workflow-code/src/records.ts b/packages/workflow-code/src/records.ts index 5f6b45ebeb7..13895a6384a 100644 --- a/packages/workflow-code/src/records.ts +++ b/packages/workflow-code/src/records.ts @@ -90,3 +90,19 @@ export function parseStoredList(raw: unknown): unknown[] | undefined { const parsed = typeof raw === 'string' ? parseJsonContainer(raw) : undefined return Array.isArray(parsed) ? parsed : undefined } + +/** Appends `item` to the list `map` holds under `key`, starting the list if there is none. */ +export function pushTo(map: Map, key: K, item: V): void { + const list = map.get(key) + if (list) list.push(item) + else map.set(key, [item]) +} + +/** Every string under `value`, depth first. */ +export function* strings(value: unknown): Generator { + if (typeof value === 'string') yield value + else if (Array.isArray(value)) for (const item of value) yield* strings(item) + else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) yield* strings(item) + } +} diff --git a/packages/workspace-code/src/kinds/table-rows.ts b/packages/workspace-code/src/kinds/table-rows.ts index 2aa21400c6b..fecda4e0e58 100644 --- a/packages/workspace-code/src/kinds/table-rows.ts +++ b/packages/workspace-code/src/kinds/table-rows.ts @@ -22,6 +22,7 @@ import { isRecordLike } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import type { Json } from '@sim/workflow-code' +import { utf8Bytes } from '@sim/workflow-code/limits' import type { ResourceFingerprint } from '../base' import { diagnostic, @@ -73,6 +74,11 @@ export function hasRowKey(row: TableRowCells, rowKey: string): boolean { return rowKeyText(row[rowKey]) !== undefined } +/** The UTF-8 bytes `row` takes as one line of a rows file, its newline included. */ +export function printedRowBytes(row: TableRowCells): number { + return utf8Bytes(JSON.stringify(filledCells(row))) + 1 +} + /** A row without its empty cells (null, `''` and `[]`, which a table stores as no value). */ function filledCells(row: TableRowCells): TableRowCells { const cells: TableRowCells = {}