diff --git a/AGENTS.md b/AGENTS.md index 1f7faa8..6980871 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -258,6 +258,7 @@ default-exports the object (safety rule 5). | Simulation | `personalityId`, `scenarioId` | Personality and scenario files | `scenarioId: books-cleaning` | | Suite | `simulationIds[]` | Simulation files | `- books-cleaning-calm` | | Any server block | `credentialId` | A credential **name** in the org | `credentialId: my-api-credential` | +| Assistant, or squad overrides | `credentialIds[]` | Credential **names**. The list is every key the call may use, not one provider's: [credential pins](docs/guides/credential-pins.md) | `- azure-south` | The engine resolves IDs and credential names to each org's UUIDs on push. diff --git a/README.md b/README.md index 7d370df..aecfe08 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,7 @@ when run without arguments. Full flags and examples: | [Writing system prompts](docs/guides/writing-prompts.md) | Structure a voice agent's prompt | | [PR checks](docs/guides/pr-checks.md) | Test every pull request with simulations | | [Promotion](docs/guides/promotion.md) | Move resources from dev to staging to production | +| [Credential pins](docs/guides/credential-pins.md) | Pin a call to specific keys with `credentialIds`, and mark an org customer-keys-only | | [How the engine works](docs/guides/how-it-works.md) | Understand sync, references, credentials and state | | [Configuration](docs/guides/configuration.md) | Environment variables and secrets | | [Troubleshooting](docs/guides/troubleshooting.md) | Fix common errors | diff --git a/docs/guides/commands.md b/docs/guides/commands.md index 414a7a8..5e6bc59 100644 --- a/docs/guides/commands.md +++ b/docs/guides/commands.md @@ -233,6 +233,7 @@ tell it. Nothing changes inside the files. | --- | --- | | `.vapi-state..json` | `state//state.json` | | `resources//.vapi-org.json` | `state//org.json` | +| `resources//.vapi-credential-providers.json` | `state//credential-providers.json` | | `.vapi-state-hash//` | `state//local/baselines/` | | `.vapi-state..snapshots/` | `state//local/snapshots/` | @@ -345,6 +346,13 @@ git add state/commit-step.json baselines yet, the first `pull` after upgrading keeps every local file, and an edit made in the dashboard comes down on the next `pull`. Nothing is lost. +- **`credential-pin-unchecked`** from `validate`, once for each org that has a + `credentialIds` list, until that org's credential providers record is + committed. It is a warning and changes no exit code. Run + `npm run pull -- --bootstrap` and commit the file it writes: + `state//credential-providers.json`, or + `resources//.vapi-credential-providers.json` while the org's state is + in the older layout. See [credential pins](credential-pins.md). After `npm run migrate`: @@ -373,6 +381,7 @@ for dir in state/*/; do org=$(basename "$dir") if [ -f "state/$org/state.json" ]; then mv "state/$org/state.json" ".vapi-state.$org.json"; fi if [ -f "state/$org/org.json" ]; then mv "state/$org/org.json" "resources/$org/.vapi-org.json"; fi + if [ -f "state/$org/credential-providers.json" ]; then mv "state/$org/credential-providers.json" "resources/$org/.vapi-credential-providers.json"; fi # This machine's baselines and snapshots. They are not in git, so everyone # who has pulled or pushed since the move runs these two on their machine. if [ -d "state/$org/local/baselines" ]; then mkdir -p .vapi-state-hash && mv "state/$org/local/baselines" ".vapi-state-hash/$org"; fi diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 79779fd..9ccd16e 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -49,6 +49,7 @@ Set these in GitHub under **Settings → Secrets and variables → Actions**. | --- | --- | | `promotion.yml` | Promotion pipelines between orgs. Start from `promotion.example.yml`. | | `vapi-checks.yml` | PR checks. Start from `vapi-checks.example.yml`. | +| `resources//.vapi-policy.yml` | Rules the org must hold, written by hand. `customerKeysOnly: true` makes a `credentialIds` list that hides one of the org's keys an error instead of a warning. See [Credential pins](credential-pins.md). | | `resources//.vapi-ignore` | Platform resources this repo shouldn't manage, as glob patterns. See `resources/.vapi-ignore.example` and [YAML conventions](../learnings/yaml-conventions.md). | These are hand-written. The files the engine writes live apart from them, in @@ -58,6 +59,7 @@ These are hand-written. The files the engine writes live apart from them, in | --- | --- | | `state//state.json` | Which platform resource (UUID) each file under `resources//` is. | | `state//org.json` | The Vapi org ID the folder belongs to. Written by the first command that can see the key's org, then checked by every command before any request: a key for another org stops the command. If a folder genuinely moves to another org, delete the file in a reviewed commit. | +| `state//credential-providers.json` | The provider of each saved credential, by name. Written by every pull and committed with the state file. It holds no secret. The credential-pin check reads it. While an org's state is in the older layout it is `resources//.vapi-credential-providers.json`. | ## Other environment variables diff --git a/docs/guides/credential-pins.md b/docs/guides/credential-pins.md new file mode 100644 index 0000000..2ba56e3 --- /dev/null +++ b/docs/guides/credential-pins.md @@ -0,0 +1,156 @@ +# Credential pins: `credentialIds` lists + +An assistant, a squad member's `assistantOverrides` and a squad's +`membersOverrides` can each carry a `credentialIds` list. It pins a call to +specific keys, for example one site's Azure OpenAI key in an org that holds a +key per site. This page explains what the list does on a call and what +`npm run validate` checks about it. + +## What a list does + +A `credentialIds` list is not a choice for one provider. It is the full set +of keys the call may use: + +- As soon as any assistant on the call has a list, the call can use only the + listed keys. Every squad member's list is pooled into one list for the + whole call, so a list on one member narrows the other members too. +- A provider whose key is not on the list does not fail. The model, voice or + transcriber runs on Vapi's own key, a tool or server request goes out + without its key, and the recording does not go to your storage bucket. +- The phone number's own credential stays usable whatever the list says. + +So a list has to name every key the call needs: the model key, the voice +key, the transcriber key, each key a tool or server names in `credentialId`, +the storage bucket key and the Langfuse key if the org has them. + +Lists combine like this: + +| Where the list is | What happens | +| --- | --- | +| On the assistant | Kept. Lists from overrides are added to it. | +| In a member's `assistantOverrides` | Used, unless the squad's `membersOverrides` also has a list, which replaces it. | +| In the squad's `membersOverrides` | Applies to every member. | + +One list per squad, in `membersOverrides`, is the simplest layout: shared +assistants carry no list, and each squad names its own keys. + +```yaml +# resources//squads/south-front-desk.yml +name: South Front Desk +members: + - assistantId: triage + - assistantId: pharmacy +membersOverrides: + credentialIds: + - azure-south # this site's Azure OpenAI key + - elevenlabs # voice + - deepgram # transcriber + - crm-webhook # the key the lookup tool names + - recordings # storage bucket +``` + +## What validate checks + +`npm run validate -- ` (and `apply`, which runs it first) reports a list +that leaves out a key **the org has saved** for something the call uses: + +| The call uses | Reported when | +| --- | --- | +| A model, voice or transcriber, or a fallback voice or transcriber | The org has a key for that provider and the list has none. An OpenAI model can run on an `openai` key or an `azure-openai` key, so either on the list is enough. Every other provider takes a key saved under its own name. | +| A tool, server or model that names a key in `credentialId` | That key is not on the list. | +| Storage | The org has a storage bucket key and the list has none. | +| Langfuse | The org has a Langfuse key and the list leaves it out. | + +It does not report a provider the org has no key of its own for. Running +that provider on Vapi's key is a normal setup, with or without a list. + +A different key for the same provider counts as a key for it. A squad pinned +to one site's Azure OpenAI key is not told about the other sites' keys. + +For a squad, the check looks at the call as a whole: the pooled list, and +what every member uses after its overrides are applied. An assistant with a +list of its own is also checked on its own, since it can be called outside a +squad. + +The finding is `credential-pin-hides-key`: + +```text +⚠️ [credential-pin-hides-key] squads/south-front-desk (credentialIds): the voice of member "triage" and the voice of member "pharmacy" use 11labs, and this org has "elevenlabs" saved for it, but the call's credentialIds list ("azure-south", "deepgram") has no key for 11labs. A call that has a credentialIds list can use only the keys on it, so 11labs would run on Vapi's own key, not yours. Add the key to the list +``` + +### Limits + +- The check reads files. A squad member that is not a file in the org's + folder (a dashboard-owned assistant) adds nothing to it. +- It knows each key's provider, not which models or regions the key covers. + A list with an Azure OpenAI key that does not serve the assistant's model + passes. +- A tool or server with no `credentialId` is not checked. +- Lists added at call time (`assistantOverrides` on a call or a phone + number) are outside the repo and are not seen. + +## Orgs that must only use their own keys + +By default the finding is a warning. For an org whose calls must never run +on Vapi's keys, add a policy file and commit it: + +```yaml +# resources//.vapi-policy.yml +customerKeysOnly: true +``` + +In that org the finding is an error: `validate` fails, `apply` stops before +it pulls or pushes, and the **Validate resources** check fails on the pull +request. `npm run push` on its own only blocks on errors with `--strict`, +like every other validation error. + +The policy belongs to one org. Promotion does not copy it, so set it in each +org that needs it. An unknown key in the file stops `validate`, so a typo +can't switch the rule off. + +The policy makes a hidden key an error. It does not require the org to have +a key for every provider it uses. + +## How the check knows what each key is for + +The state file maps a credential's name to its UUID and nothing else. Pull +also records each key's provider in +`state//credential-providers.json`: + +```json +{ + "azure-south": { "provider": "azure-openai" }, + "elevenlabs": { "provider": "11labs" }, + "recordings": { "bucket": true, "provider": "azure" } +} +``` + +Commit it with the state file. It holds no secret. Every pull rewrites it, +including `npm run pull -- --bootstrap`, which changes no resource +file. If the file is missing or older than the state file, the check reports +`credential-pin-unchecked` and asks for that pull. + +In a repository whose state is still in the layout from before `state/`, the +file is `resources//.vapi-credential-providers.json` instead, which the +commit step that repository already has stages with the folder. `npm run +migrate` moves it with the rest: see +[Upgrading from an older version](commands.md#upgrading-from-an-older-version). + +Give every key a name in Vapi before the first pull. Two keys with no name +and the same provider get the same name here, and only one is kept. Pull +warns when that happens. + +## Promotion + +A credential with the binding policy `omit` is dropped from a promoted file. +For a single `credentialId` that is the point. For an entry in a +`credentialIds` list it narrows what calls in the destination may use, so +the plan says so: + +```text +release: dev → prod + update squads/south-front-desk.yml + ⚠️ squads/south-front-desk.yml: credentialIds entry "elevenlabs" is left out in prod (binding policy "omit"). Calls there can use only the keys still on the list, so that provider would run on Vapi's own key. +``` + +See [Promotion](promotion.md) for bindings. diff --git a/docs/guides/how-it-works.md b/docs/guides/how-it-works.md index ca78a0a..807b66b 100644 --- a/docs/guides/how-it-works.md +++ b/docs/guides/how-it-works.md @@ -160,6 +160,11 @@ Credentials are managed automatically through the state file. No secrets in reso 2. Resource files use human-readable credential names 3. **Push** resolves names back to UUIDs before sending to the API +Pull also records each credential's provider in +`state//credential-providers.json`, so `validate` can check a +`credentialIds` list against the keys the org has: see +[Credential pins](credential-pins.md). + Setup and pull also refresh a marked block in the gitignored `.env.` file: ```dotenv diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index 1d9bae1..1404879 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -21,6 +21,9 @@ left alone. 5. Choose `bind` or `omit` for credentials and phone numbers in each target org. Pull generates stable aliases in `.env.` from resources that already exist there; promotion never copies secrets or provisions numbers. + An `omit` that drops an entry from a `credentialIds` list is flagged in the + plan, because it narrows which keys calls in the target may use: see + [Credential pins](credential-pins.md#promotion). 6. Run the read-only plan for each transition before the first apply and confirm every create, update, and delete is expected. diff --git a/docs/learnings/squads.md b/docs/learnings/squads.md index 8184080..2a8ec6a 100644 --- a/docs/learnings/squads.md +++ b/docs/learnings/squads.md @@ -165,6 +165,10 @@ The final assistant configuration is built by merging these layers in order: Later layers win on conflicts. `variableValues` from all layers are merged separately. +`credentialIds` follows the same order with one difference: a list on the base assistant is kept and the override's list is added to it, while `squad.membersOverrides.credentialIds` replaces `member.assistantOverrides.credentialIds`. + +**Gotcha:** a `credentialIds` list anywhere in the squad narrows the whole call to the listed keys, for every member and every provider. A provider left off runs on Vapi's own key with no error. List every key the call needs, and let `npm run validate` check it: see [Credential pins](../guides/credential-pins.md). + **Gotcha:** Liquid template substitution replaces undefined variables with **empty strings**, not errors. If a variable isn't in the merged `variableValues`, any `{{ myVar }}` reference silently becomes `""`. --- diff --git a/improvements.md b/improvements.md index c7a2ce6..3f4090f 100644 --- a/improvements.md +++ b/improvements.md @@ -110,6 +110,7 @@ you which stack PR closes the row.** | 37 | Resource validation ran only at deploy time, after merge | A config `apply` refuses could merge and block deploys and promotion | #32 | RESOLVED 2026-10-03 (#76) | | 38 | Promotion can't carry a simulation that uses a stock personality by UUID | Gated orgs, and any promoted tests, need local personality files | None | Open | | 39 | Engine state looked like temp files and sat in three places | Deleting the state file makes the next push duplicate every resource | None | RESOLVED 2026-11-17 (#97) | +| 40 | Nothing checked a `credentialIds` list against the keys an org has saved | A list that hides the org's own key runs the call on Vapi's key, silently | None | RESOLVED 2026-11-17 (#96) | **Active backlog after cleanup:** `#2`, `#6`, `#8`, `#12`, `#20`, `#24–#26`, `#31`, and the open remainder of `#27` (wiring the listing-completeness verdict into push/delete/audit, and moving `cleanup.ts` onto the shared pager). Resolved entries stay in this file as historical incident notes per the maintenance directive; stale superseded backlog rows are not duplicated. @@ -2203,6 +2204,64 @@ run's hand-off to the workflow, and an older workflow reads it there. --- +## 40. Nothing checked a `credentialIds` list against the keys an org has saved + +**[RESOLVED 2026-11-17] (#96)** + +**Discovered:** 2026-11-17, while planning one squad per site in an org that +holds an Azure OpenAI key per site. + +### Problem + +A `credentialIds` list is the full set of keys a call may use, and every +squad member's list is pooled into one. A provider left off the list does +not fail the call: it runs on Vapi's own key. The engine could not tell a +complete list from one that hides the org's own voice, transcriber or +storage key. + +### Current behavior (Verified) + +- The state file records `name → uuid` for a credential and nothing else, + so the engine did not know which provider a saved key is for. +- `validate` only checked that each name on a list exists in the state file. +- Promotion dropped a `credentialIds` entry whose binding policy is `omit` + without saying so. +- Two credentials with no name and the same provider get the same slug on + pull, and the second silently replaces the first in the state file. + +### Risk + +An org that must run only on its own keys sends audio or prompts to Vapi's +provider accounts when one entry is missed in one squad file, with nothing +in the push output or the call to show it. + +### Current mitigation + +None before this change, other than reading every list by hand. + +### Possible fix + +Record each credential's provider on pull, report a list that leaves out a +key the org has saved for something the call uses, and let an org mark +itself customer-keys-only so the report is an error there. + +### Status + +**RESOLVED 2026-11-17.** Pull writes +`state//credential-providers.json` +(`resources//.vapi-credential-providers.json` while the org's state is +in the older layout) and warns when two +credentials collide on one name. `validate` has the +`credential-pin-hides-key` rule, an error in an org whose +`.vapi-policy.yml` sets `customerKeysOnly: true`. The promotion plan warns +when `omit` drops a `credentialIds` entry. See +[Credential pins](docs/guides/credential-pins.md). + +Still open: the slug collision itself is only warned about, and the check +does not know which models or regions a key covers. + +--- + ## Out of scope (intentionally not improvements) - **State file is identity-only and not git-ignored.** It's intentionally diff --git a/src/credential-kinds.ts b/src/credential-kinds.ts new file mode 100644 index 0000000..1f894c1 --- /dev/null +++ b/src/credential-kinds.ts @@ -0,0 +1,94 @@ +// What each saved credential is a key for. +// +// The state file maps a credential's name to its UUID and nothing else. A +// `credentialIds` list can only be checked against the org's keys when the +// provider of each key is known too, and pull already fetches that. Pull +// records it in state//credential-providers.json, which is committed +// with the state file. It holds no secret: a provider name per key, and +// whether a storage bucket is set on it. +// +// Config-free so validate, push and tests can read any org. + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, relative } from "node:path"; +import { stateRecordPath } from "./state-paths.ts"; +import { sortedKeysReplacer } from "./state-serialize.ts"; + +// state//credential-providers.json, or +// resources//.vapi-credential-providers.json while the org's state is +// still in the old layout, where an older commit step stages it. +export function credentialKindsPath(rootDir: string, org: string): string { + return stateRecordPath(rootDir, org, "credential-providers"); +} + +/** The file as a person names it: its path from the repository root. */ +export function credentialKindsLabel(rootDir: string, org: string): string { + return relative(rootDir, credentialKindsPath(rootDir, org)).replaceAll( + "\\", + "/", + ); +} + +export interface CredentialKind { + provider: string; + // The key has a storage bucket configured. A gcp, azure or supabase key + // only receives recordings when it does. + bucket?: true; +} + +// Credential name (as resource files write it) → what it is a key for. +export type CredentialKinds = Record; + +function record(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +/** What one credential from GET /credential is a key for. */ +export function credentialKindOf( + credential: Record, +): CredentialKind | undefined { + if (typeof credential.provider !== "string" || !credential.provider) + return undefined; + const plan = record(credential.bucketPlan); + const bucket = plan?.name || plan?.containerName; + return typeof bucket === "string" && bucket + ? { provider: credential.provider, bucket: true } + : { provider: credential.provider }; +} + +/** The org's recorded credential kinds, or undefined before its first pull. */ +export function credentialKindsRead( + rootDir: string, + org: string, +): CredentialKinds | undefined { + const path = credentialKindsPath(rootDir, org); + if (!existsSync(path)) return undefined; + const raw = record(JSON.parse(readFileSync(path, "utf-8"))); + if (!raw) throw new Error(`${path} must hold a JSON object`); + const kinds: CredentialKinds = {}; + for (const [name, value] of Object.entries(raw)) { + const entry = record(value); + if (typeof entry?.provider !== "string") continue; + kinds[name] = + entry.bucket === true + ? { provider: entry.provider, bucket: true } + : { provider: entry.provider }; + } + return kinds; +} + +/** Write the org's credential kinds. Returns false when nothing changed. */ +export function credentialKindsWrite( + rootDir: string, + org: string, + kinds: CredentialKinds, +): boolean { + const path = credentialKindsPath(rootDir, org); + const content = `${JSON.stringify(kinds, sortedKeysReplacer, 2)}\n`; + if (existsSync(path) && readFileSync(path, "utf-8") === content) return false; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); + return true; +} diff --git a/src/org-policy.ts b/src/org-policy.ts new file mode 100644 index 0000000..4c243d2 --- /dev/null +++ b/src/org-policy.ts @@ -0,0 +1,47 @@ +// Rules an org's folder must hold, set by hand in +// resources//.vapi-policy.yml and committed with the folder. +// +// customerKeysOnly: true +// +// marks an org whose calls must never run on Vapi's own provider keys. The +// credential-pin check is an error there and a warning everywhere else. +// Promotion skips dotfiles, so a policy never travels to another org. +// +// Config-free so validate, push and tests can read any org. + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; + +export const ORG_POLICY_FILE = ".vapi-policy.yml"; + +export interface OrgPolicy { + customerKeysOnly: boolean; +} + +const ORG_POLICY_KEYS = ["customerKeysOnly"]; + +export function orgPolicyParse(content: string): OrgPolicy { + const raw: unknown = parseYaml(content) ?? {}; + if (typeof raw !== "object" || Array.isArray(raw)) + throw new Error(`${ORG_POLICY_FILE} must be a mapping`); + const policy = raw as Record; + // A typo (`customerKeyOnly:`) must not quietly turn a safety rule off. + for (const key of Object.keys(policy)) + if (!ORG_POLICY_KEYS.includes(key)) + throw new Error( + `${ORG_POLICY_FILE}: unknown key "${key}" (known: ${ORG_POLICY_KEYS.join(", ")})`, + ); + const { customerKeysOnly = false } = policy; + if (typeof customerKeysOnly !== "boolean") + throw new Error( + `${ORG_POLICY_FILE}: customerKeysOnly must be true or false`, + ); + return { customerKeysOnly }; +} + +/** The org's policy; every rule is off when the file doesn't exist. */ +export function orgPolicyRead(resourcesDir: string): OrgPolicy { + const path = join(resourcesDir, ORG_POLICY_FILE); + return orgPolicyParse(existsSync(path) ? readFileSync(path, "utf-8") : ""); +} diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index f1cc39b..cdf06b3 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -271,6 +271,7 @@ async function transitionRun( ); for (const change of plan.changes) console.log(` ${change.kind.padEnd(6)} ${change.path}`); + for (const warning of plan.warnings) console.warn(` ⚠️ ${warning}`); if (plan.changes.length === 0) console.log(" no changes"); if (plan.changes.length === 0) return false; const check = gates.checks.get(transition.source); diff --git a/src/promotion.ts b/src/promotion.ts index eb13bae..f27b52b 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -55,6 +55,8 @@ export interface PromotionPlan { rootDir: string; target: string; changes: PromotionChange[]; + // Things a reviewer should know that don't stop the promotion. + warnings: string[]; } export interface PromotionPlanOptions { @@ -396,6 +398,8 @@ export function promotionBindingsApply( bindings: PromotionBindings, resolved: PromotionBindingsResolved, targetState: StateFile, + // Collects the credentialIds entries the "omit" policy left out. + omittedFromLists: string[] = [], ): unknown { function credential(ref: string): string | undefined { const alias = resolved.credentialReverse.get(ref) ?? ref; @@ -436,10 +440,15 @@ export function promotionBindingsApply( continue; } if (key === "credentialIds" && Array.isArray(child)) { - output[key] = child - .filter((item): item is string => typeof item === "string") - .map(credential) - .filter((item): item is string => item !== undefined); + const kept: string[] = []; + for (const item of child) { + if (typeof item !== "string") continue; + const bound = credential(item); + if (bound) kept.push(bound); + else + omittedFromLists.push(resolved.credentialReverse.get(item) ?? item); + } + output[key] = kept; continue; } if (key === "phoneNumberIds" && Array.isArray(child)) { @@ -599,14 +608,17 @@ export async function promotionPlanBuild( const targetByPath = new Map(target.map((file) => [file.path, file])); const sourcePaths = new Set([...wanted.values()].map((file) => file.path)); const changes: PromotionChange[] = []; + const warnings: string[] = []; for (const file of wanted.values()) { const parsed = await resourceData(file, options.rootDir, options.source); const canonical = referencesCanonicalize(parsed.data, options.sourceState); + const omitted: string[] = []; const transformed = promotionBindingsApply( canonical, bindings, resolved, options.targetState, + omitted, ); if ( extname(file.path) === ".ts" && @@ -621,12 +633,19 @@ export async function promotionPlanBuild( ? file.content : renderContent(transformed, file.path, parsed.body); const existing = targetByPath.get(file.path); - if (!existing || existing.content !== content) + if (!existing || existing.content !== content) { changes.push({ kind: existing ? "update" : "create", path: file.path, content, }); + // A credentialIds list is the only set of keys a call may use, so an + // entry left out of it is not neutral the way an omitted credentialId is. + for (const alias of new Set(omitted)) + warnings.push( + `${file.path}: credentialIds entry "${alias}" is left out in ${options.target} (binding policy "omit"). Calls there can use only the keys still on the list, so that provider would run on Vapi's own key.`, + ); + } } for (const file of target) if ( @@ -636,7 +655,12 @@ export async function promotionPlanBuild( ) changes.push({ kind: "delete", path: file.path }); changes.sort((left, right) => left.path.localeCompare(right.path)); - return { rootDir: options.rootDir, target: options.target, changes }; + return { + rootDir: options.rootDir, + target: options.target, + changes, + warnings, + }; } export async function promotionPlanApply(plan: PromotionPlan): Promise { diff --git a/src/pull.ts b/src/pull.ts index ec7d2a3..bf4215b 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -26,6 +26,11 @@ import { VAPI_ENV, VAPI_TOKEN, } from "./config.ts"; +import { + type CredentialKinds, + credentialKindOf, + credentialKindsWrite, +} from "./credential-kinds.ts"; import { credentialReverseMap } from "./credentials.ts"; import { classifyDrift, @@ -338,19 +343,32 @@ async function pullCredentials(state: StateFile): Promise { existingReverse.set(entry.uuid, slug); } + const kinds: CredentialKinds = {}; for (const cred of credentials) { // Reuse existing slug if available, otherwise generate a new one let slug = existingReverse.get(cred.id); if (!slug) { slug = credentialSlug(cred); } + // Two keys with the same name (or none, and the same provider) get the + // same slug, and the later one replaces the earlier in state. + const twin = newSection[slug]; + if (twin) + console.warn( + ` ⚠️ Credentials ${twin.uuid} and ${cred.id} are both named "${slug}"; only the second is kept. Give each a name of its own in Vapi, then pull again.`, + ); newSection[slug] = existingReverse.has(cred.id) ? state.credentials[slug]! : { uuid: cred.id, origin: "pulled" }; + const kind = credentialKindOf(cred); + if (kind) kinds[slug] = kind; console.log(` 🔑 ${slug} -> ${cred.id}`); } state.credentials = newSection; + // The state file stays a name → UUID map. What each key is for goes in a + // file of its own, which the credential-pin check reads. + credentialKindsWrite(BASE_DIR, VAPI_ENV, kinds); } // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/push.ts b/src/push.ts index c507dbd..3fb2ef4 100644 --- a/src/push.ts +++ b/src/push.ts @@ -26,6 +26,7 @@ import { checkDriftForUpdate, type DriftCheckResult } from "./drift.ts"; import { deleteBaseline, writeBaseline } from "./hash-store.ts"; import { prepareResourceLinks } from "./link-preflight.ts"; import { NameCollisionReport } from "./name-collisions.ts"; +import { orgPolicyRead } from "./org-policy.ts"; import { computePushPlan, pushBlockingFindings } from "./push-plan.ts"; import { reconcileStateKeyForResource } from "./reconcile-state-key.ts"; import { writeSnapshot } from "./snapshot.ts"; @@ -37,6 +38,7 @@ import { validateNoIgnoredReferences, validateResources, } from "./validate.ts"; +import { validateCredentialPins } from "./validate-credential-pins.ts"; import { validateReferences } from "./validate-refs.ts"; import { changedFieldPaths, @@ -57,6 +59,10 @@ const RESOURCE_LABEL_TO_TYPE: Record = { "simulation suite": "simulationSuites", }; +import { + credentialKindsLabel, + credentialKindsRead, +} from "./credential-kinds.ts"; import { credentialForwardMap, credentialReverseMap, @@ -1797,6 +1803,14 @@ async function main(): Promise { state, ignorePatterns: loadIgnorePatterns(), }), + ...validateCredentialPins({ + loaded: loadedResources, + org: VAPI_ENV, + state, + kinds: credentialKindsRead(BASE_DIR, VAPI_ENV), + kindsFile: credentialKindsLabel(BASE_DIR, VAPI_ENV), + customerKeysOnly: orgPolicyRead(RESOURCES_DIR).customerKeysOnly, + }), ]; if (findings.length > 0) { console.log(summarizeFindings(findings)); diff --git a/src/state-paths.ts b/src/state-paths.ts index 3032c1d..effa760 100644 --- a/src/state-paths.ts +++ b/src/state-paths.ts @@ -2,6 +2,7 @@ // // state//state.json resource name → Vapi UUID (committed) // state//org.json the Vapi org this folder is pinned to (committed) +// state//.json what the engine records beside them (committed) // state//local/baselines/ drift baselines, one file per UUID (this machine only) // state//local/snapshots/ push snapshots for rollback (this machine only) // state/commit-step.json a person's word that their commit step stages state/ (committed) @@ -184,6 +185,41 @@ export function orgPinFileRead(rootDir: string, org: string): string { /** The org pin to write: the one that is read. */ export const orgPinFileWrite = orgPinFileRead; +/** The records the engine keeps beside an org's state file, by name. */ +export const STATE_RECORDS = ["credential-providers"] as const; +export type StateRecord = (typeof STATE_RECORDS)[number]; + +function recordPath(rootDir: string, org: string, name: StateRecord): string { + return join(stateOrgDir(rootDir, org), `${name}.json`); +} + +function recordLegacyPath( + rootDir: string, + org: string, + name: StateRecord, +): string { + return join(rootDir, "resources", org, `.vapi-${name}.json`); +} + +/** + * A record the engine keeps beside the state file: where it is, else where + * its first write goes. For an org in the old layout that is + * resources//.vapi-.json, beside the old org pin, which a commit + * step from before state/ already stages with the folder. + */ +export function stateRecordPath( + rootDir: string, + org: string, + name: StateRecord, +): string { + return fileInLayout( + recordPath(rootDir, org, name), + recordLegacyPath(rootDir, org, name), + rootDir, + org, + ); +} + /** The state file as a person names it: its path from the repository root. */ export function stateFileLabel(rootDir: string, org: string): string { return relative(rootDir, stateFileRead(rootDir, org)).replaceAll("\\", "/"); @@ -237,6 +273,12 @@ function committedFiles(rootDir: string, org: string): CommittedFile[] { legacyLabel: `resources/${org}/.vapi-org.json`, nextLabel: `${STATE_DIR}/${org}/org.json`, }, + ...STATE_RECORDS.map((name) => ({ + legacy: recordLegacyPath(rootDir, org, name), + next: recordPath(rootDir, org, name), + legacyLabel: `resources/${org}/.vapi-${name}.json`, + nextLabel: `${STATE_DIR}/${org}/${name}.json`, + })), ]; } diff --git a/src/validate-cmd.ts b/src/validate-cmd.ts index 67ee142..681c630 100644 --- a/src/validate-cmd.ts +++ b/src/validate-cmd.ts @@ -8,7 +8,17 @@ import { existsSync } from "fs"; import { relative, resolve } from "path"; import { fileURLToPath } from "url"; -import { BASE_DIR, loadIgnorePatterns, VAPI_ENV } from "./config.ts"; +import { + BASE_DIR, + loadIgnorePatterns, + RESOURCES_DIR, + VAPI_ENV, +} from "./config.ts"; +import { + credentialKindsLabel, + credentialKindsRead, +} from "./credential-kinds.ts"; +import { orgPolicyRead } from "./org-policy.ts"; import { loadResources } from "./resources.ts"; import { createEmptyState, loadState } from "./state.ts"; import { stateFileRead } from "./state-paths.ts"; @@ -19,6 +29,7 @@ import { validateNoIgnoredReferences, validateResources, } from "./validate.ts"; +import { validateCredentialPins } from "./validate-credential-pins.ts"; import { validateReferences } from "./validate-refs.ts"; async function main(): Promise { @@ -53,6 +64,9 @@ async function main(): Promise { if (!stateExists) console.log("📄 No state file yet: references must name local files."); const state = stateExists ? loadState() : createEmptyState(); + const policy = orgPolicyRead(RESOURCES_DIR); + if (policy.customerKeysOnly) + console.log("🔐 Customer keys only: a hidden key is an error in this org."); const findings = [ ...validateResources(resources), ...validateNoIgnoredReferences(resources, ignorePatterns), @@ -62,6 +76,14 @@ async function main(): Promise { state, ignorePatterns, }), + ...validateCredentialPins({ + loaded: resources, + org: VAPI_ENV, + state, + kinds: credentialKindsRead(BASE_DIR, VAPI_ENV), + kindsFile: credentialKindsLabel(BASE_DIR, VAPI_ENV), + customerKeysOnly: policy.customerKeysOnly, + }), ]; console.log(`\n${summarizeFindings(findings)}\n`); diff --git a/src/validate-credential-pins.ts b/src/validate-credential-pins.ts new file mode 100644 index 0000000..aafacda --- /dev/null +++ b/src/validate-credential-pins.ts @@ -0,0 +1,347 @@ +// ───────────────────────────────────────────────────────────────────────────── +// Credential pins: does a `credentialIds` list hide a key the org has saved? +// +// On a call, `credentialIds` is not a choice for one provider. As soon as any +// assistant in the call lists any id, the listed ids are the only keys the +// whole call may use, and every squad member's list is pooled into one. A +// provider whose key is left off does not fail: it runs on Vapi's own key, a +// tool request goes out without its key, and recordings skip the org's bucket. +// +// This check fires only when the list leaves out a key the org HAS saved for +// something the call uses. An org with no key of its own for a provider is +// relying on Vapi's on purpose and is left alone. A phone number's own +// credential stays usable whatever the list says, so it is never reported. +// +// Config-free: the caller passes the state, the credential kinds and whether +// the org is customer-keys-only (a finding is then an error, else a warning). +// ───────────────────────────────────────────────────────────────────────────── + +import type { CredentialKinds } from "./credential-kinds.ts"; +import { credentialReverseMap } from "./credentials.ts"; +import { referencesCollect } from "./resolver.ts"; +import type { LoadedResources, ResourceType, StateFile } from "./types.ts"; +import type { ValidationFinding } from "./validate.ts"; + +type Data = Record; + +// Key providers that can serve a model provider. A provider not listed here +// takes a key saved under its own name, as voices and transcribers do. +const MODEL_KEY_PROVIDERS: Record = { + openai: ["openai", "azure-openai"], +}; + +// Storage keys. Every call uploads its recording and other files with the +// first one it may use; gcp, azure and supabase keys only count as storage +// when a bucket is set on them. +const STORAGE_PROVIDERS = new Set(["s3", "cloudflare", "s3-compatible", "aws"]); +const STORAGE_PROVIDERS_WITH_BUCKET = new Set(["gcp", "azure", "supabase"]); + +const record = (value: unknown): Data | undefined => + value && typeof value === "object" && !Array.isArray(value) + ? (value as Data) + : undefined; +const list = (value: unknown): unknown[] => (Array.isArray(value) ? value : []); +const refClean = (id: string) => id.split("##")[0]?.trim() ?? ""; +const quoted = (names: Iterable) => + [...names].map((name) => `"${name}"`).join(", "); + +// Overrides laid over an assistant the way the platform lays them: mappings +// merge key by key, lists are joined (so the assistant's own credentialIds +// stay and the override's are added), and a block for a different provider +// replaces the assistant's block. +function overlay(current: unknown, override: unknown): unknown { + if (override === undefined) return current; + if (Array.isArray(override)) return [...list(current), ...override]; + const over = record(override); + if (!over) return override; + if (Object.keys(over).length === 0) return current ?? override; + const base = record(current); + if (!base) return override; + if ( + typeof over.provider === "string" && + typeof base.provider === "string" && + over.provider !== base.provider + ) + return override; + const merged: Data = { ...base }; + for (const [key, value] of Object.entries(over)) + merged[key] = overlay(base[key], value); + return merged; +} + +// A member's overrides: the squad's membersOverrides replace the member's +// own assistantOverrides key by key, except tools:append, which is joined. +function memberOverrides(member: Data, squad: Data): Data { + const own = record(member.assistantOverrides) ?? {}; + const shared = record(squad.membersOverrides) ?? {}; + const merged: Data = { ...own, ...shared }; + const appended = [ + ...list(own["tools:append"]), + ...list(shared["tools:append"]), + ]; + if (appended.length > 0) merged["tools:append"] = appended; + return merged; +} + +// Every `credentialId` value under a resource: a tool, a server, a model. +function credentialRefs(value: unknown, refs = new Set()): Set { + if (Array.isArray(value)) { + for (const item of value) credentialRefs(item, refs); + return refs; + } + for (const [key, child] of Object.entries(record(value) ?? {})) { + if (key === "credentialIds") continue; + if (key !== "credentialId") credentialRefs(child, refs); + else if (typeof child === "string" && child) refs.add(child); + } + return refs; +} + +interface ProviderUse { + what: string; + provider: string; + keyProviders: string[]; +} + +function providerUses(assistant: Data): ProviderUse[] { + const uses: ProviderUse[] = []; + const add = ( + what: string, + block: unknown, + keyProviders: Record = {}, + ) => { + const provider = record(block)?.provider; + if (typeof provider === "string" && provider) + uses.push({ + what, + provider, + keyProviders: keyProviders[provider] ?? [provider], + }); + }; + const voice = record(assistant.voice); + const transcriber = record(assistant.transcriber); + add("model", assistant.model, MODEL_KEY_PROVIDERS); + add("voice", voice); + for (const fallback of list(record(voice?.fallbackPlan)?.voices)) + add("fallback voice", fallback); + add("transcriber", transcriber); + for (const fallback of list(record(transcriber?.fallbackPlan)?.transcribers)) + add("fallback transcriber", fallback); + return uses; +} + +// One assistant as it runs on a call, and who to name in a message. +interface Participant { + label: string; + assistant: Data; +} + +interface CallTarget { + type: ResourceType; + resourceId: string; + participants: Participant[]; +} + +export function validateCredentialPins(args: { + loaded: LoadedResources; + org: string; + state: StateFile; + // What each saved credential is a key for; undefined before the first pull + // that recorded it. + kinds: CredentialKinds | undefined; + // The file the kinds are read from, as a person names it. + kindsFile: string; + customerKeysOnly: boolean; +}): ValidationFinding[] { + const { loaded, org, state, kinds, kindsFile, customerKeysOnly } = args; + const severity = customerKeysOnly ? "error" : "warn"; + const uuidToName = credentialReverseMap(state); + const nameOf = (ref: string) => uuidToName.get(ref) ?? ref; + const toolUuidToId = new Map( + Object.entries(state.tools).map(([id, entry]) => [entry.uuid, id]), + ); + const tools = new Map(loaded.tools.map((tool) => [tool.resourceId, tool])); + const assistants = new Map( + loaded.assistants.map((assistant) => [assistant.resourceId, assistant]), + ); + + const targets: CallTarget[] = []; + for (const assistant of loaded.assistants) + targets.push({ + type: "assistants", + resourceId: assistant.resourceId, + participants: [{ label: "this assistant", assistant: assistant.data }], + }); + for (const squad of loaded.squads) { + const participants: Participant[] = []; + for (const item of list(squad.data.members)) { + const member = record(item); + if (!member) continue; + const id = + typeof member.assistantId === "string" + ? refClean(member.assistantId) + : undefined; + // A member that isn't a file here (a dashboard-owned assistant) can't + // be read, so it adds nothing to the check. + const base = record(member.assistant) ?? (id && assistants.get(id)?.data); + if (!base) continue; + const name = id ?? (typeof base.name === "string" ? base.name : "inline"); + participants.push({ + label: `member "${name}"`, + assistant: overlay(base, memberOverrides(member, squad.data)) as Data, + }); + } + targets.push({ + type: "squads", + resourceId: squad.resourceId, + participants, + }); + } + + const findings: ValidationFinding[] = []; + const missingKinds = Object.keys(state.credentials).filter( + (name) => !kinds?.[name], + ); + let kindsReported = false; + + for (const target of targets) { + const pin = new Set( + target.participants + .flatMap((participant) => list(participant.assistant.credentialIds)) + .filter((id): id is string => typeof id === "string" && id !== "") + .map(nameOf), + ); + if (pin.size === 0) continue; + const pinText = quoted(pin); + const finding = (rule: string, message: string) => + findings.push({ + severity, + type: target.type, + resourceId: target.resourceId, + rule, + message, + fieldPath: "credentialIds", + }); + const onlyListed = + "A call that has a credentialIds list can use only the keys on it"; + + // Keys named outright: a tool's or a server's credentialId. + const named = new Map>(); + const name = (ref: string, who: string) => { + const key = nameOf(ref); + if (!pin.has(key)) named.set(key, (named.get(key) ?? new Set()).add(who)); + }; + for (const { label, assistant } of target.participants) { + for (const ref of credentialRefs(assistant)) name(ref, label); + for (const ref of referencesCollect(assistant).get("tools") ?? []) { + const id = toolUuidToId.get(refClean(ref)) ?? refClean(ref); + for (const toolRef of credentialRefs(tools.get(id)?.data)) + name(toolRef, `tool "${id}"`); + } + } + for (const [key, who] of named) + finding( + "credential-pin-hides-key", + `${[...who].join(" and ")} ${who.size > 1 ? "name" : "names"} ` + + `credential "${key}", but the call's ` + + `credentialIds list (${pinText}) leaves it out. ${onlyListed}, so ` + + `that request would go out without the key. Add "${key}" to the list`, + ); + + // The rest needs each key's provider. Without it, say so once and stop. + const unknown = [...pin].filter((key) => !kinds?.[key]); + if (!kinds || missingKinds.length > 0 || unknown.length > 0) { + if (unknown.length > 0 && kinds && missingKinds.length === 0) + finding( + "credential-pin-unchecked", + `credentialIds lists ${quoted(unknown)}, which this repo has no ` + + `record of, so the list can't be checked against the keys this ` + + `org has saved. Check the name, or run \`npm run pull -- ${org} ` + + `--bootstrap\` if the key was just created`, + ); + else if (!kindsReported) + finding( + "credential-pin-unchecked", + `${kindsFile} ` + + (kinds + ? `has no entry for ${quoted(missingKinds)}` + : "doesn't exist yet") + + `, so credentialIds lists in this org can't be checked against ` + + `the keys it has saved. Run \`npm run pull -- ${org} --bootstrap\` ` + + `and commit that file`, + ); + kindsReported = true; + continue; + } + + const pinnedProviders = new Set( + [...pin].map((key) => kinds[key]!.provider), + ); + for (const { assistant } of target.participants) + for (const inline of list(assistant.credentials)) { + const provider = record(inline)?.provider; + if (typeof provider === "string") pinnedProviders.add(provider); + } + const savedFor = (providers: string[]) => + Object.keys(kinds).filter((key) => + providers.includes(kinds[key]!.provider), + ); + + // Model, voice and transcriber, with their fallbacks. + const hidden = new Map< + string, + { provider: string; keys: string[]; uses: Set } + >(); + for (const { label, assistant } of target.participants) + for (const use of providerUses(assistant)) { + if (use.keyProviders.some((provider) => pinnedProviders.has(provider))) + continue; + const keys = savedFor(use.keyProviders); + if (keys.length === 0) continue; + const entry = hidden.get(use.provider) ?? { + provider: use.provider, + keys, + uses: new Set(), + }; + entry.uses.add(`the ${use.what} of ${label}`); + hidden.set(use.provider, entry); + } + for (const { provider, keys, uses } of hidden.values()) + finding( + "credential-pin-hides-key", + `${[...uses].join(" and ")} ${uses.size > 1 ? "use" : "uses"} ` + + `${provider}, and this org has ` + + `${quoted(keys)} saved for it, but the call's credentialIds list ` + + `(${pinText}) has no key for ${provider}. ${onlyListed}, so ` + + `${provider} would run on Vapi's own key, not yours. Add the key ` + + `to the list`, + ); + + // Keys every call uses when the org has them. + const storage = Object.keys(kinds).filter((key) => { + const kind = kinds[key]!; + return ( + STORAGE_PROVIDERS.has(kind.provider) || + (STORAGE_PROVIDERS_WITH_BUCKET.has(kind.provider) && kind.bucket) + ); + }); + if (storage.length > 0 && !storage.some((key) => pin.has(key))) + finding( + "credential-pin-hides-key", + `this org has a storage bucket key saved (${quoted(storage)}), but ` + + `the call's credentialIds list (${pinText}) has none. ` + + `${onlyListed}, so its recording and other call files would not ` + + `go to your bucket. Add the key to the list`, + ); + const langfuse = savedFor(["langfuse"]); + if (langfuse.length > 0 && !pinnedProviders.has("langfuse")) + finding( + "credential-pin-hides-key", + `this org has a Langfuse key saved (${quoted(langfuse)}), but the ` + + `call's credentialIds list (${pinText}) leaves it out. ` + + `${onlyListed}, so the call would not be traced. Add the key to ` + + `the list`, + ); + } + return findings; +} diff --git a/state/README.md b/state/README.md index f3e5a8d..322f120 100644 --- a/state/README.md +++ b/state/README.md @@ -6,6 +6,7 @@ The engine writes everything in this folder. Don't edit or delete these files by | --- | --- | --- | | `state//state.json` | Which Vapi resource (UUID) each file under `resources//` is. | Yes | | `state//org.json` | The Vapi org that `resources//` belongs to. A command run with another org's key stops before it reads or changes anything. | Yes | +| `state//credential-providers.json` | The provider of each saved credential, by name, with no secret. The [credential-pin check](../docs/guides/credential-pins.md) reads it. | Yes | | `state//local/baselines/` | The last version of each resource this machine saw on the platform, used to notice dashboard edits. | No | | `state//local/snapshots/` | A copy of each resource taken before a push, for `npm run rollback`. | No | | `state/commit-step.json` | Only in a repository whose own pipeline commits state in a way the engine can't read. It is that repository's word, given with `npm run migrate -- --commit-step-stages-state`, that the pipeline stages this folder. | Yes | @@ -14,15 +15,18 @@ The engine writes everything in this folder. Don't edit or delete these files by - **`state.json`**: the engine no longer knows which resources already exist, so the next push creates every one of them again as a duplicate. Restore the file from git (`git checkout -- state//state.json`). Never rebuild it by hand. - **`org.json`**: the next command pins the folder again to whichever org its key belongs to, so the protection against a wrong key is gone until then. Restore it from git. +- **`credential-providers.json`**: `validate` can't check `credentialIds` lists and says so. The next pull writes the file again. - **`commit-step.json`**: nothing is lost. A CI run under that pipeline stops before it changes anything and says how to put the file back. - **`local/`**: nothing breaks. The next pull records new baselines. Snapshots taken so far are gone, so there is nothing to roll back to until the next push. ## What is yours to edit -Everything under `resources//`, including `resources//.vapi-ignore`, is hand-written and yours. This folder is the engine's. +Everything under `resources//`, including `resources//.vapi-ignore` and `resources//.vapi-policy.yml`, is hand-written and yours. This folder is the engine's. ## Repositories created before this folder existed The same files used to sit elsewhere: `.vapi-state..json` and `.vapi-state..snapshots/` at the repository root, `.vapi-state-hash//`, and `resources//.vapi-org.json`. The engine keeps reading and writing them there, on your machine and in CI, for as long as you leave them. +While an org's state is there, the engine writes its credential providers record to `resources//.vapi-credential-providers.json`, where the commit step such a repository already has stages it. + Only `npm run migrate` moves them here. Before it moves anything it says what the step that commits state after a run has to stage, and it moves nothing while it can't see that step stage this folder, unless you tell it yours does. Commit the removal of the old files together with the new folder. See [Upgrading from an older version](../docs/guides/commands.md#upgrading-from-an-older-version). diff --git a/tests/credential-kinds.test.ts b/tests/credential-kinds.test.ts new file mode 100644 index 0000000..0425b60 --- /dev/null +++ b/tests/credential-kinds.test.ts @@ -0,0 +1,145 @@ +import "./engine-sandbox.ts"; +import assert from "node:assert/strict"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import test from "node:test"; +import { + credentialKindOf, + credentialKindsLabel, + credentialKindsRead, + credentialKindsWrite, +} from "../src/credential-kinds.ts"; +import { migrateAll } from "../src/migrate-hash-store.ts"; +import { orgPolicyParse, orgPolicyRead } from "../src/org-policy.ts"; +import { commitStepAcknowledge } from "../src/state-paths.ts"; +import { stateFixturePath } from "./state-fixture.ts"; + +test("a credential's kind is its provider, and whether a bucket is set", () => { + assert.deepEqual(credentialKindOf({ provider: "11labs", apiKey: "…" }), { + provider: "11labs", + }); + assert.deepEqual( + credentialKindOf({ provider: "gcp", bucketPlan: { name: "calls" } }), + { provider: "gcp", bucket: true }, + ); + assert.deepEqual( + credentialKindOf({ + provider: "azure", + bucketPlan: { containerName: "calls" }, + }), + { provider: "azure", bucket: true }, + ); + // A speech key: same provider, no bucket. + assert.deepEqual(credentialKindOf({ provider: "azure", bucketPlan: {} }), { + provider: "azure", + }); + assert.equal(credentialKindOf({}), undefined); +}); + +test("the kinds file round-trips, holds nothing else, and isn't rewritten unchanged", () => { + const dir = mkdtempSync(join(tmpdir(), "vapi-kinds-")); + try { + writeFileSync(stateFixturePath(dir, "clinic"), "{}\n"); + assert.equal(credentialKindsRead(dir, "clinic"), undefined); + const kinds = { + recordings: { provider: "azure", bucket: true as const }, + elevenlabs: { provider: "11labs" }, + }; + assert.equal(credentialKindsWrite(dir, "clinic", kinds), true); + const file = join(dir, "state/clinic/credential-providers.json"); + assert.equal( + readFileSync(file, "utf-8"), + `{ + "elevenlabs": { + "provider": "11labs" + }, + "recordings": { + "bucket": true, + "provider": "azure" + } +} +`, + ); + assert.deepEqual(credentialKindsRead(dir, "clinic"), kinds); + const written = statSync(file).mtimeMs; + assert.equal(credentialKindsWrite(dir, "clinic", kinds), false); + assert.equal(statSync(file).mtimeMs, written); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("while an org's state is in the older layout the kinds file sits where that layout's commit step stages it, and migrate moves it", async () => { + const dir = mkdtempSync(join(tmpdir(), "vapi-kinds-")); + try { + writeFileSync(join(dir, ".vapi-state.clinic.json"), "{}\n"); + const kinds = { elevenlabs: { provider: "11labs" } }; + assert.equal(credentialKindsWrite(dir, "clinic", kinds), true); + const beside = join( + dir, + "resources/clinic/.vapi-credential-providers.json", + ); + const inFolder = join(dir, "state/clinic/credential-providers.json"); + assert.equal(credentialKindsLabel(dir, "clinic"), relative(dir, beside)); + assert.equal(existsSync(beside), true); + // Nothing of this org's is under state/, which that step doesn't stage. + assert.equal(existsSync(join(dir, "state")), false); + assert.deepEqual(credentialKindsRead(dir, "clinic"), kinds); + + commitStepAcknowledge(dir); + await migrateAll(dir); + assert.equal(existsSync(beside), false); + assert.equal(credentialKindsLabel(dir, "clinic"), relative(dir, inFolder)); + assert.deepEqual(credentialKindsRead(dir, "clinic"), kinds); + // And the next write goes to the folder. + assert.equal( + credentialKindsWrite(dir, "clinic", { + deepgram: { provider: "deepgram" }, + }), + true, + ); + assert.equal(existsSync(beside), false); + assert.match(readFileSync(inFolder, "utf-8"), /deepgram/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("an org is customer-keys-only only when its policy file says so", () => { + assert.deepEqual(orgPolicyParse(""), { customerKeysOnly: false }); + assert.deepEqual(orgPolicyParse("customerKeysOnly: true\n"), { + customerKeysOnly: true, + }); + assert.deepEqual(orgPolicyParse("# nothing set\n"), { + customerKeysOnly: false, + }); + const dir = mkdtempSync(join(tmpdir(), "vapi-policy-")); + try { + assert.deepEqual(orgPolicyRead(dir), { customerKeysOnly: false }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a policy file that could be a typo is refused, not ignored", () => { + assert.throws( + () => orgPolicyParse("customerKeyOnly: true\n"), + /unknown key "customerKeyOnly" \(known: customerKeysOnly\)/, + ); + assert.throws( + () => orgPolicyParse('customerKeysOnly: "yes"\n'), + /customerKeysOnly must be true or false/, + ); + assert.throws( + () => orgPolicyParse("- customerKeysOnly\n"), + /must be a mapping/, + ); +}); diff --git a/tests/credential-pins-cli.test.ts b/tests/credential-pins-cli.test.ts new file mode 100644 index 0000000..56336d8 --- /dev/null +++ b/tests/credential-pins-cli.test.ts @@ -0,0 +1,253 @@ +import "./engine-sandbox.ts"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { Worker } from "node:worker_threads"; + +// End to end, through the commands a person runs: pull records what each +// saved key is for, and validate (which apply runs first) then catches a +// credentialIds list that hides one. The API is a local stub. + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const ORG = "clinic"; + +// The stub must live on its own thread: spawnSync parks this one. +function startStub( + credentials: unknown[], +): Promise<{ worker: Worker; port: number }> { + return new Promise((resolveStart, rejectStart) => { + const source = ` + const http = require('node:http'); + const { parentPort, workerData } = require('node:worker_threads'); + const server = http.createServer((req, res) => { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + const isCredentials = (req.url || '').startsWith('/credential'); + res.end(JSON.stringify(isCredentials ? workerData.credentials : [])); + }); + server.listen(0, '127.0.0.1', () => { + parentPort.postMessage({ port: server.address().port }); + }); + parentPort.on('message', () => server.close(() => process.exit(0))); + `; + const worker = new Worker(source, { + eval: true, + workerData: { credentials }, + }); + worker.once("error", rejectStart); + worker.once("message", (message: { port: number }) => + resolveStart({ worker, port: message.port }), + ); + }); +} + +const id = (n: number) => + `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; + +// What GET /credential returns for the org. Secrets are made up, and none of +// them may reach a file. +const CREDENTIALS = [ + { + id: id(1), + orgId: "org-1", + provider: "azure-openai", + name: "azure-north", + openAIKey: "secret-1", + }, + { + id: id(2), + orgId: "org-1", + provider: "azure-openai", + name: "azure-south", + openAIKey: "secret-2", + }, + { + id: id(3), + orgId: "org-1", + provider: "11labs", + name: "elevenlabs", + apiKey: "secret-3", + }, + { id: id(4), orgId: "org-1", provider: "deepgram", apiKey: "secret-4" }, + { + id: id(5), + orgId: "org-1", + provider: "azure", + name: "recordings", + bucketPlan: { containerName: "calls", connectionString: "secret-5" }, + }, + // Two keys with no name and the same provider end up with one name. + { id: id(6), orgId: "org-1", provider: "deepgram", apiKey: "secret-6" }, +]; + +const TRIAGE = `--- +name: Triage +model: + provider: openai + model: gpt-4.1 +voice: + provider: 11labs + voiceId: sarah +transcriber: + provider: deepgram + model: nova-3 +--- + +You are the triage assistant. +`; + +const squad = (credentialIds: string[]) => + `name: Front Desk +members: + - assistantId: triage +membersOverrides: + credentialIds: +${credentialIds.map((name) => ` - ${name}`).join("\n")} +`; + +function run(dir: string, port: number, script: string, ...args: string[]) { + return spawnSync("node", ["--import", "tsx", `src/${script}`, ORG, ...args], { + cwd: dir, + env: { + ...process.env, + VAPI_TOKEN: "fake-token-not-used", + VAPI_BASE_URL: `http://127.0.0.1:${port}`, + GITHUB_ACTIONS: "", + }, + encoding: "utf-8", + timeout: 60_000, + }); +} + +// Where the engine keeps the two files it writes, in each layout. A fresh +// fork of this repository has a workflow that stages state/, so its state +// starts there; a repository whose commit step can't be seen keeps the older +// places, which that step already stages. +const LAYOUTS = [ + { + name: "state//", + workflows: true, + stateFile: `state/${ORG}/state.json`, + kindsFile: `state/${ORG}/credential-providers.json`, + }, + { + name: "the older layout", + workflows: false, + stateFile: `.vapi-state.${ORG}.json`, + kindsFile: `resources/${ORG}/.vapi-credential-providers.json`, + }, +]; + +for (const layout of LAYOUTS) + test(`pull records each key's provider, and validate catches a list that hides one (${layout.name})`, async () => { + const dir = mkdtempSync(join(tmpdir(), "vapi-credential-pins-")); + if (layout.workflows) + cpSync(join(REPO_ROOT, ".github"), join(dir, ".github"), { + recursive: true, + }); + cpSync(join(REPO_ROOT, "src"), join(dir, "src"), { recursive: true }); + cpSync(join(REPO_ROOT, "package.json"), join(dir, "package.json")); + symlinkSync( + join(REPO_ROOT, "node_modules"), + join(dir, "node_modules"), + "dir", + ); + const orgDir = join(dir, "resources", ORG); + mkdirSync(join(orgDir, "assistants"), { recursive: true }); + mkdirSync(join(orgDir, "squads"), { recursive: true }); + writeFileSync(join(orgDir, "assistants", "triage.md"), TRIAGE); + const squadPath = join(orgDir, "squads", "front-desk.yml"); + // The practice's model key, the transcriber key and the bucket: no voice key. + writeFileSync(squadPath, squad(["azure-south", "deepgram", "recordings"])); + + const { worker, port } = await startStub(CREDENTIALS); + try { + // Before any pull there is nothing to check the list against. + const before = run(dir, port, "validate-cmd.ts"); + assert.equal(before.status, 0, before.stdout + before.stderr); + assert.match(before.stdout, /credential-pin-unchecked/); + assert.ok( + before.stdout.includes(`${layout.kindsFile} doesn't exist yet`), + before.stdout, + ); + + const pull = run(dir, port, "pull.ts", "--bootstrap"); + assert.equal(pull.status, 0, pull.stdout + pull.stderr); + assert.match( + pull.stdout + pull.stderr, + new RegExp( + `Credentials ${id(4)} and ${id(6)} are both named "deepgram"; only the second is kept`, + ), + ); + const kindsText = readFileSync(join(dir, layout.kindsFile), "utf-8"); + assert.deepEqual(JSON.parse(kindsText), { + "azure-north": { provider: "azure-openai" }, + "azure-south": { provider: "azure-openai" }, + deepgram: { provider: "deepgram" }, + elevenlabs: { provider: "11labs" }, + recordings: { provider: "azure", bucket: true }, + }); + assert.doesNotMatch(kindsText, /secret/); + // The state file is still a name → UUID map and nothing else. + const state = JSON.parse( + readFileSync(join(dir, layout.stateFile), "utf-8"), + ); + assert.deepEqual(state.credentials.elevenlabs, { + uuid: id(3), + origin: "pulled", + }); + + // A warning by default: validate passes and says what is hidden. + const warned = run(dir, port, "validate-cmd.ts"); + assert.equal(warned.status, 0, warned.stdout + warned.stderr); + assert.match(warned.stdout, /0 error\(s\), 1 warning\(s\)/); + assert.match( + warned.stdout, + /credential-pin-hides-key[\s\S]*the voice of member "triage" uses 11labs, and this org has "elevenlabs" saved for it/, + ); + + // An error once the org is marked customer-keys-only: validate fails, + // which is the step apply runs before it pulls or pushes anything. + writeFileSync( + join(orgDir, ".vapi-policy.yml"), + "customerKeysOnly: true\n", + ); + const refused = run(dir, port, "validate-cmd.ts"); + assert.equal(refused.status, 1, refused.stdout + refused.stderr); + assert.match(refused.stdout, /1 error\(s\), 0 warning\(s\)/); + assert.match(refused.stderr, /Validation failed with 1 error/); + + // Naming the voice key on the list clears it. + writeFileSync( + squadPath, + squad(["azure-south", "deepgram", "recordings", "elevenlabs"]), + ); + const fixed = run(dir, port, "validate-cmd.ts"); + assert.equal(fixed.status, 0, fixed.stdout + fixed.stderr); + assert.match(fixed.stdout, /Validation passed/); + + // A misspelt policy key stops validate; it doesn't switch the rule off. + writeFileSync( + join(orgDir, ".vapi-policy.yml"), + "customerKeyOnly: true\n", + ); + const typo = run(dir, port, "validate-cmd.ts"); + assert.equal(typo.status, 1); + assert.match(typo.stderr, /unknown key "customerKeyOnly"/); + } finally { + worker.postMessage("shutdown"); + rmSync(dir, { recursive: true, force: true }); + } + }); diff --git a/tests/promotion.test.ts b/tests/promotion.test.ts index 6fc041f..996b620 100644 --- a/tests/promotion.test.ts +++ b/tests/promotion.test.ts @@ -285,6 +285,70 @@ test("promotion canonicalizes source UUIDs and applies credential and phone poli assert.match(content, /phoneNumberIds:\n\s+- target-phone/); assert.doesNotMatch(content, new RegExp(sourceTool)); assert.doesNotMatch(content, new RegExp(sourceCredential)); + // The entry left out of the list is named, with the file it was in. + assert.equal(plan.warnings.length, 1); + assert.match( + plan.warnings[0]!, + /^assistants\/a\.md: credentialIds entry "unknown" is left out in target \(binding policy "omit"\)/, + ); + } finally { + await fx.cleanup(); + } +}); + +test("promotion warns about a credentialIds entry it leaves out, and only then", async () => { + const fx = await fixture(); + try { + const sourceVoice = "33333333-3333-3333-3333-333333333333"; + await put( + fx.root, + "resources/source/squads/front-desk.yml", + `name: Front Desk\nmembers: []\nmembersOverrides:\n credentialIds: [${sourceVoice}, llm]\n server:\n url: https://example.com\n credentialId: hooks\n`, + ); + const build = (aliases: Record) => + promotionPlanBuild({ + rootDir: fx.root, + source: "source", + target: "target", + patterns: ["squads/**"], + sourceState: state({ + credentials: { + voice: { uuid: sourceVoice }, + llm: { uuid: "source-llm" }, + hooks: { uuid: "source-hooks" }, + }, + }), + targetState: state({ + credentials: { + voice: { uuid: "target-voice" }, + llm: { uuid: "target-llm" }, + hooks: { uuid: "target-hooks" }, + }, + }), + bindings: { + credentials: { default: "bind", aliases }, + phoneNumbers: { default: "omit", aliases: {} }, + }, + }); + + // Every entry bound: nothing to say. + assert.deepEqual((await build({})).warnings, []); + // A single credentialId left out is what "omit" is for: no warning. + assert.deepEqual((await build({ hooks: "omit" })).warnings, []); + // A list entry left out narrows what the call may use: warn, by name, + // even though the source wrote it as a UUID. + const plan = await build({ voice: "omit" }); + assert.match(plan.changes[0]?.content ?? "", /credentialIds:\n\s+- llm\n/); + assert.deepEqual(plan.warnings, [ + 'squads/front-desk.yml: credentialIds entry "voice" is left out in target (binding policy "omit"). Calls there can use only the keys still on the list, so that provider would run on Vapi\'s own key.', + ]); + + // Once the target file holds the promoted content there is no change, + // and no warning to repeat. + await promotionPlanApply(plan); + const again = await build({ voice: "omit" }); + assert.deepEqual(again.changes, []); + assert.deepEqual(again.warnings, []); } finally { await fx.cleanup(); } diff --git a/tests/push-preflight.test.ts b/tests/push-preflight.test.ts index a77769c..56e57d3 100644 --- a/tests/push-preflight.test.ts +++ b/tests/push-preflight.test.ts @@ -17,6 +17,7 @@ import { fileURLToPath } from "node:url"; import { computePushPlan, pushBlockingFindings } from "../src/push-plan.ts"; import type { LoadedResources, ResourceType, StateFile } from "../src/types.ts"; import { VALID_RESOURCE_TYPES } from "../src/types.ts"; +import { validateCredentialPins } from "../src/validate-credential-pins.ts"; process.argv = ["node", "test", "preflight"]; process.env.VAPI_TOKEN = "fake"; @@ -129,18 +130,71 @@ test("plan: squad auto-applied assistant errors block, including existing assist assert.deepEqual(pushBlockingFindings([], new Set(), false), []); }); +test("credential-pin severity and scope survive push preflight", () => { + const local = loaded({ + assistants: { + bot: { credentialIds: ["model"], voice: { provider: "11labs" } }, + }, + squads: { team: { members: [{ assistantId: "bot" }] } }, + }); + const tracked = state(); + tracked.credentials = { model: { uuid: UUID }, voice: { uuid: "voice-id" } }; + const kinds = { + model: { provider: "openai" }, + voice: { provider: "11labs" }, + }; + for (const customerKeysOnly of [false, true]) { + for (const recorded of [kinds, undefined]) { + const findings = validateCredentialPins({ + loaded: local, + org: "preflight", + state: tracked, + kinds: recorded, + kindsFile: "state/preflight/credential-providers.json", + customerKeysOnly, + }); + assert.equal(findings.length, recorded ? 2 : 1); + assert.ok( + findings.every( + (finding) => + finding.rule === + (recorded + ? "credential-pin-hides-key" + : "credential-pin-unchecked"), + ), + ); + const plan = computePushPlan( + { ...loaded(), squads: local.squads }, + local, + tracked, + ); + assert.equal( + pushBlockingFindings(findings, plan, false).length, + customerKeysOnly ? findings.length : 0, + ); + assert.deepEqual(pushBlockingFindings(findings, new Set(), false), []); + assert.equal( + pushBlockingFindings(findings, new Set(), true).length, + customerKeysOnly ? findings.length : 0, + ); + } + } + assert.deepEqual(pushBlockingFindings([], new Set(), true), []); +}); + async function runFixture(options: { scoped?: boolean; dryRun?: boolean; credential?: boolean; strict?: boolean; + credentialPin?: boolean; }) { const dir = mkdtempSync(join(tmpdir(), "push-preflight-")); const mutations: string[] = []; const good = { id: UUID, name: "Good" }; const bad = { id: "b0000000-0000-4000-8000-000000000002", - name: options.credential ? "Bad" : BAD, + name: options.credential || options.credentialPin ? "Bad" : BAD, ...(options.credential ? { server: { url: "https://example.com", credentialId: "missing" } } : {}), @@ -173,11 +227,25 @@ async function runFixture(options: { writeFileSync(join(folder, "good.yml"), "name: Good\n"); writeFileSync( join(folder, "zbad.yml"), - `name: ${bad.name}\n${options.credential ? "server:\n url: https://example.com\n credentialId: missing\n" : ""}`, + `name: ${bad.name}\n${options.credential ? "server:\n url: https://example.com\n credentialId: missing\n" : ""}${options.credentialPin ? "credentialIds:\n - unused\nvoice:\n provider: 11labs\n voiceId: sarah\n" : ""}`, ); const initial = state(); initial.assistants.good = { uuid: good.id }; initial.assistants.zbad = { uuid: bad.id }; + if (options.credentialPin) { + initial.credentials.voice = { uuid: "voice-id" }; + writeFileSync( + join(dir, "resources/preflight/.vapi-policy.yml"), + "customerKeysOnly: true\n", + ); + writeFileSync( + join(dir, "resources/preflight/.vapi-credential-providers.json"), + JSON.stringify({ + unused: { provider: "openai" }, + voice: { provider: "11labs" }, + }), + ); + } writeFileSync( join(dir, ".vapi-state.preflight.json"), JSON.stringify(initial), @@ -225,6 +293,12 @@ for (const [label, options] of [ ["dry-run", { dryRun: true }], ["unresolved credential", { credential: true }], ["strict scoped push", { scoped: true, strict: true }], + ["credential pin", { credentialPin: true }], + ["credential pin dry-run", { credentialPin: true, dryRun: true }], + [ + "credential pin strict scoped push", + { credentialPin: true, scoped: true, strict: true }, + ], ] as const) { test(`preflight blocks ${label} before any mutation`, async () => { const result = await runFixture(options); @@ -233,6 +307,8 @@ for (const [label, options] of [ assert.deepEqual(result.mutations, []); if ("credential" in options) assert.match(result.output, /unresolved-credential/); + if ("credentialPin" in options) + assert.match(result.output, /credential-pin-hides-key/); }); } diff --git a/tests/state-records-third-round.test.ts b/tests/state-records-third-round.test.ts index 8379d68..364e45f 100644 --- a/tests/state-records-third-round.test.ts +++ b/tests/state-records-third-round.test.ts @@ -24,7 +24,8 @@ import { fileURLToPath } from "node:url"; // engine keeps files of its own in that folder, and none of them is a // person's edit: // -// resources//.vapi-org.json shown as deleted between migrate and its commit +// resources//.vapi-org.json shown as deleted between migrate and its commit +// resources//.vapi-credential-providers.json written by every pull // // If pull counted one as a local edit, the keep rule would be off and pull // would overwrite the committed change with the platform's copy. @@ -182,6 +183,18 @@ test("older layout: an org pin the engine wrote, not committed yet, is not a loc assert.equal(kept, true, "pull overwrote the merged change"); }); +test("older layout: a credential providers record the engine wrote, not committed yet, is not a local edit", async () => { + // The first pull after upgrading leaves this file untracked. + const kept = await pullKeepsMerged(freshClone("root"), (root) => + put( + root, + "resources/dev/.vapi-credential-providers.json", + '{\n "crm-api": {\n "provider": "webhook"\n }\n}\n', + ), + ); + assert.equal(kept, true, "pull overwrote the merged change"); +}); + test("unchanged: a person's own uncommitted file in that folder still switches the keep rule off", async () => { // Only the engine's files are left out. Anything else under // resources// counts as before, .vapi-ignore included. diff --git a/tests/validate-credential-pins.test.ts b/tests/validate-credential-pins.test.ts new file mode 100644 index 0000000..c2c237b --- /dev/null +++ b/tests/validate-credential-pins.test.ts @@ -0,0 +1,574 @@ +import "./engine-sandbox.ts"; +import assert from "node:assert/strict"; +import test from "node:test"; +import type { CredentialKinds } from "../src/credential-kinds.ts"; +import type { LoadedResources, ResourceType, StateFile } from "../src/types.ts"; +import { validateCredentialPins } from "../src/validate-credential-pins.ts"; + +// A credentialIds list is the only set of keys a call may use. The check +// fires when the list leaves out a key the org has saved for something the +// call uses, and stays quiet when the org has no key of its own for it. + +const uuid = (n: number) => + `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; + +// The keys a clinic group might save: one Azure OpenAI key per practice, and +// one of everything else. +const KINDS: CredentialKinds = { + "azure-north": { provider: "azure-openai" }, + "azure-south": { provider: "azure-openai" }, + elevenlabs: { provider: "11labs" }, + deepgram: { provider: "deepgram" }, + "crm-webhook": { provider: "webhook" }, + recordings: { provider: "azure", bucket: true }, + "main-line": { provider: "twilio" }, +}; +const FULL_PIN = ["elevenlabs", "deepgram", "crm-webhook", "recordings"]; + +function loaded( + files: Partial>>, +): LoadedResources { + const list = (type: ResourceType) => + Object.entries(files[type] ?? {}).map(([resourceId, data]) => ({ + resourceId, + filePath: `/repo/resources/clinic/${type}/${resourceId}.yml`, + data: data as Record, + })); + return { + tools: list("tools"), + structuredOutputs: list("structuredOutputs"), + assistants: list("assistants"), + squads: list("squads"), + personalities: list("personalities"), + scenarios: list("scenarios"), + simulations: list("simulations"), + simulationSuites: list("simulationSuites"), + evals: list("evals"), + }; +} + +function state(kinds: CredentialKinds, tools: string[] = []): StateFile { + const section = (ids: string[], offset: number) => + Object.fromEntries(ids.map((id, i) => [id, { uuid: uuid(offset + i) }])); + return { + credentials: section(Object.keys(kinds), 100), + tools: section(tools, 200), + assistants: {}, + structuredOutputs: {}, + squads: {}, + personalities: {}, + scenarios: {}, + simulations: {}, + simulationSuites: {}, + evals: {}, + }; +} + +// Each finding as "severity rule type/resource: message". +function check( + files: Partial>>, + options: { + kinds?: CredentialKinds | null; + customerKeysOnly?: boolean; + stateKinds?: CredentialKinds; + } = {}, +): string[] { + const kinds = options.kinds === null ? undefined : (options.kinds ?? KINDS); + return validateCredentialPins({ + loaded: loaded(files), + org: "clinic", + state: state( + options.stateKinds ?? kinds ?? KINDS, + Object.keys(files.tools ?? {}), + ), + kinds, + kindsFile: "state/clinic/credential-providers.json", + customerKeysOnly: options.customerKeysOnly ?? false, + }).map( + (f) => `${f.severity} ${f.rule} ${f.type}/${f.resourceId}: ${f.message}`, + ); +} + +const triage = { + name: "Triage", + model: { provider: "openai", model: "gpt-4.1" }, + voice: { provider: "11labs", voiceId: "sarah" }, + transcriber: { provider: "deepgram", model: "nova-3" }, +}; +const pharmacy = { ...triage, name: "Pharmacy" }; + +const frontDesk = (membersOverrides: object, members?: object[]) => ({ + name: "Front Desk", + members: members ?? [{ assistantId: "triage" }, { assistantId: "pharmacy" }], + membersOverrides, +}); + +test("no credentialIds list, no finding, whatever keys the org has", () => { + assert.deepEqual( + check({ + assistants: { triage, pharmacy }, + squads: { "front-desk": frontDesk({}) }, + }), + [], + ); +}); + +test("a list that names every key the call uses passes", () => { + assert.deepEqual( + check({ + assistants: { triage, pharmacy }, + squads: { + "front-desk": frontDesk({ + credentialIds: ["azure-south", ...FULL_PIN], + }), + }, + }), + [], + ); +}); + +test("a squad list that leaves out the saved voice key is reported once, for every member", () => { + const found = check({ + assistants: { triage, pharmacy }, + squads: { + "front-desk": frontDesk({ + credentialIds: ["azure-south", "deepgram", "crm-webhook", "recordings"], + }), + }, + }); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /^warn credential-pin-hides-key squads\/front-desk: /, + ); + assert.match( + found[0]!, + /the voice of member "triage" and the voice of member "pharmacy" use 11labs, and this org has "elevenlabs" saved for it/, + ); + assert.match(found[0]!, /11labs would run on Vapi's own key, not yours/); +}); + +test("it is an error in an org marked customer-keys-only", () => { + const found = check( + { + assistants: { triage, pharmacy }, + squads: { "front-desk": frontDesk({ credentialIds: ["azure-south"] }) }, + }, + { customerKeysOnly: true }, + ); + // Voice, transcriber and the storage bucket are all hidden. + assert.equal(found.length, 3); + for (const line of found) + assert.match(line, /^error credential-pin-hides-key squads\/front-desk: /); +}); + +test("a provider the org has no key of its own for is left alone", () => { + // No ElevenLabs or Deepgram key saved: those run on Vapi's keys on purpose. + const kinds: CredentialKinds = { + "azure-south": { provider: "azure-openai" }, + }; + assert.deepEqual( + check( + { + assistants: { triage, pharmacy }, + squads: { "front-desk": frontDesk({ credentialIds: ["azure-south"] }) }, + }, + { kinds, customerKeysOnly: true }, + ), + [], + ); +}); + +test("another key for the same provider counts: one practice's key, not the other's", () => { + assert.deepEqual( + check({ + assistants: { triage, pharmacy }, + squads: { + "front-desk": frontDesk({ + credentialIds: ["azure-north", ...FULL_PIN], + }), + }, + }), + [], + ); +}); + +test("an OpenAI model is served by an openai key or an azure-openai key", () => { + const kinds: CredentialKinds = { + "azure-south": { provider: "azure-openai" }, + "openai-direct": { provider: "openai" }, + elevenlabs: { provider: "11labs" }, + }; + const assistant = { ...triage, transcriber: undefined }; + const withPin = (credentialIds: string[]) => + check( + { assistants: { triage: { ...assistant, credentialIds } } }, + { kinds }, + ); + // Either kind of key on the list is a key for the model. + assert.deepEqual(withPin(["azure-south", "elevenlabs"]), []); + assert.deepEqual(withPin(["openai-direct", "elevenlabs"]), []); + // Neither on the list: both saved keys are hidden. + const found = withPin(["elevenlabs"]); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /the model of this assistant uses openai, and this org has "azure-south", "openai-direct" saved for it/, + ); +}); + +test("a member's own assistantOverrides list is replaced by the squad's", () => { + // The member names the voice key, but membersOverrides wins key by key. + const found = check({ + assistants: { triage }, + squads: { + "front-desk": frontDesk( + { credentialIds: ["azure-south", "deepgram", "recordings"] }, + [ + { + assistantId: "triage", + assistantOverrides: { credentialIds: ["elevenlabs"] }, + }, + ], + ), + }, + }); + assert.equal(found.length, 1); + assert.match(found[0]!, /uses 11labs/); +}); + +test("an assistant's own list is kept, and the squad's is added to it", () => { + const found = check({ + assistants: { + triage: { + ...triage, + credentialIds: ["azure-south", "elevenlabs", "recordings"], + }, + }, + squads: { + "front-desk": frontDesk({ credentialIds: ["deepgram"] }, [ + { assistantId: "triage" }, + ]), + }, + }); + // In the squad the two lists together cover the call. Called on its own, + // the assistant has only its own list, which has no transcriber key. + assert.equal(found.length, 1); + assert.match( + found[0]!, + /^warn credential-pin-hides-key assistants\/triage: the transcriber of this assistant uses deepgram/, + ); +}); + +test("one member's list narrows the whole call", () => { + // Only pharmacy has a list, yet triage's voice loses its key as well. + const found = check( + { + assistants: { + triage, + pharmacy: { + ...pharmacy, + credentialIds: [ + "azure-south", + "deepgram", + "elevenlabs", + "recordings", + ], + }, + scheduler: { + name: "Scheduler", + model: { provider: "anthropic", model: "claude" }, + voice: { provider: "cartesia", voiceId: "x" }, + }, + }, + squads: { + "front-desk": frontDesk({}, [ + { assistantId: "triage" }, + { assistantId: "pharmacy" }, + { assistantId: "scheduler" }, + ]), + }, + }, + { + kinds: { ...KINDS, "cartesia-key": { provider: "cartesia" } }, + }, + ); + assert.deepEqual( + found.map((line) => line.replace(/, and this org[\s\S]*$/, "")), + [ + 'warn credential-pin-hides-key squads/front-desk: the voice of member "scheduler" uses cartesia', + ], + ); +}); + +test("an override that changes the provider is what gets checked", () => { + const found = check( + { + assistants: { triage }, + squads: { + "front-desk": frontDesk( + { + credentialIds: [ + "azure-south", + "elevenlabs", + "deepgram", + "recordings", + ], + voice: { provider: "cartesia", voiceId: "x" }, + }, + [{ assistantId: "triage" }], + ), + }, + }, + { kinds: { ...KINDS, "cartesia-key": { provider: "cartesia" } } }, + ); + assert.equal(found.length, 1); + assert.match(found[0]!, /the voice of member "triage" uses cartesia/); +}); + +test("fallback voices and transcribers are checked like the primary", () => { + const found = check({ + assistants: { + triage: { + ...triage, + voice: { + provider: "vapi", + voiceId: "Elliot", + fallbackPlan: { voices: [{ provider: "11labs", voiceId: "sarah" }] }, + }, + credentialIds: ["azure-south", "deepgram", "recordings"], + }, + }, + }); + assert.equal(found.length, 1); + assert.match(found[0]!, /the fallback voice of this assistant uses 11labs/); +}); + +test("a key a tool or server names must be on the list", () => { + const lookup = { + type: "apiRequest", + url: "https://example.com/lookup", + credentialId: "crm-webhook", + }; + const files = (credentialIds: string[]) => ({ + tools: { lookup }, + assistants: { + triage: { + ...triage, + model: { ...triage.model, toolIds: ["lookup"] }, + server: { + url: "https://example.com/events", + credentialId: "crm-webhook", + }, + credentialIds, + }, + }, + }); + assert.deepEqual( + check( + files([ + "azure-south", + "elevenlabs", + "deepgram", + "recordings", + "crm-webhook", + ]), + ), + [], + ); + const found = check( + files(["azure-south", "elevenlabs", "deepgram", "recordings"]), + ); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /this assistant and tool "lookup" name credential "crm-webhook", but the call's credentialIds list .* leaves it out/, + ); + assert.match(found[0]!, /that request would go out without the key/); +}); + +test("a tool appended in an override is read too", () => { + const found = check({ + assistants: { triage }, + squads: { + "front-desk": frontDesk( + { + credentialIds: [ + "azure-south", + "elevenlabs", + "deepgram", + "recordings", + ], + "tools:append": [ + { + type: "function", + server: { + url: "https://example.com", + credentialId: "crm-webhook", + }, + }, + ], + }, + [{ assistantId: "triage" }], + ), + }, + }); + assert.equal(found.length, 1); + assert.match(found[0]!, /member "triage" names credential "crm-webhook"/); +}); + +test("the storage bucket key is used by every call", () => { + const pinned = { + ...triage, + credentialIds: ["azure-south", "elevenlabs", "deepgram"], + }; + const found = check({ assistants: { triage: pinned } }); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /this org has a storage bucket key saved \("recordings"\).*would not go to your bucket/, + ); + // An azure, gcp or supabase key without a bucket is not storage. + assert.deepEqual( + check( + { assistants: { triage: pinned } }, + { kinds: { ...KINDS, recordings: { provider: "azure" } } }, + ), + [], + ); + // An s3 key always is. + assert.equal( + check( + { assistants: { triage: pinned } }, + { kinds: { ...KINDS, recordings: { provider: "s3" } } }, + ).length, + 1, + ); +}); + +test("a saved Langfuse key left off the list is reported", () => { + const found = check( + { + assistants: { + triage: { ...triage, credentialIds: ["azure-south", ...FULL_PIN] }, + }, + }, + { kinds: { ...KINDS, tracing: { provider: "langfuse" } } }, + ); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /Langfuse key saved \("tracing"\).*would not be traced/, + ); +}); + +test("a phone number's credential never has to be listed", () => { + // The call keeps its phone number's key whatever the list says. + assert.deepEqual( + check( + { + assistants: { + triage: { + name: "Triage", + model: { provider: "vapi" }, + voice: { provider: "11labs", voiceId: "sarah" }, + credentialIds: ["elevenlabs"], + }, + }, + }, + { + kinds: { + elevenlabs: { provider: "11labs" }, + "main-line": { provider: "twilio" }, + trunk: { provider: "byo-sip-trunk" }, + }, + customerKeysOnly: true, + }, + ), + [], + ); +}); + +test("a key written inline on the assistant covers its provider", () => { + assert.deepEqual( + check({ + assistants: { + triage: { + ...triage, + credentials: [{ provider: "11labs", apiKey: "…" }], + credentialIds: ["azure-south", "deepgram", "recordings"], + }, + }, + }), + [], + ); +}); + +test("a list written as UUIDs is read through the state file", () => { + const names = Object.keys(KINDS); + const id = (name: string) => uuid(100 + names.indexOf(name)); + assert.deepEqual( + check({ + assistants: { + triage: { + ...triage, + credentialIds: ["azure-south", ...FULL_PIN].map(id), + }, + }, + }), + [], + ); +}); + +test("without the credential kinds file the check says so once, and still checks named keys", () => { + const pinned = (name: string) => ({ + ...triage, + name, + server: { url: "https://example.com", credentialId: "crm-webhook" }, + credentialIds: ["azure-south"], + }); + const found = check( + { assistants: { triage: pinned("Triage"), pharmacy: pinned("Pharmacy") } }, + { kinds: null, customerKeysOnly: true }, + ); + assert.deepEqual( + found.map((line) => line.split(": ")[0]), + [ + "error credential-pin-hides-key assistants/triage", + "error credential-pin-unchecked assistants/triage", + "error credential-pin-hides-key assistants/pharmacy", + ], + ); + assert.match( + found[1]!, + /state\/clinic\/credential-providers\.json doesn't exist yet.*npm run pull -- clinic --bootstrap/, + ); +}); + +test("a credential newer than the kinds file makes the check ask for a pull", () => { + const found = check( + { assistants: { triage: { ...triage, credentialIds: ["azure-south"] } } }, + { + kinds: { "azure-south": { provider: "azure-openai" } }, + stateKinds: KINDS, + }, + ); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /credential-pin-unchecked .* has no entry for "azure-north"/, + ); +}); + +test("a name on the list that nothing knows is reported, not guessed at", () => { + const found = check({ + assistants: { + triage: { ...triage, credentialIds: ["azure-suoth", ...FULL_PIN] }, + }, + }); + assert.equal(found.length, 1); + assert.match( + found[0]!, + /credential-pin-unchecked assistants\/triage: credentialIds lists "azure-suoth", which this repo has no record of/, + ); +});