diff --git a/.depcheckrc.yml b/.depcheckrc.yml index ef2dad39f..5191edc71 100644 --- a/.depcheckrc.yml +++ b/.depcheckrc.yml @@ -50,6 +50,10 @@ ignores: # Used by @ocap/nodejs to build the sqlite3 bindings - 'node-gyp' + # Peer dependency of @metamask/kernel-shims/endoify-node which should be + # listed as a dev or prod dependency of consuming packages + - '@libp2p/webrtc' + # These are peer dependencies of various modules we actually do # depend on, which have been elevated to full dependencies (even # though we don't actually depend on them) in order to work around a @@ -68,3 +72,6 @@ ignores: # Testing # This import is used in files which are meant to fail - 'does-not-exist' + +ignore-patterns: + - dist/ diff --git a/.gitignore b/.gitignore index 1279a097f..8e2319873 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .DS_Store dist/ coverage/ -docs/ +docs/* !docs/*.md # Logs diff --git a/docs/kernel-to-host-captp.md b/docs/kernel-to-host-captp.md new file mode 100644 index 000000000..cc559dbea --- /dev/null +++ b/docs/kernel-to-host-captp.md @@ -0,0 +1,243 @@ +# Kernel-to-Host CapTP Serialization Flow + +This document explains the serialization pipeline between the kernel and host application, covering how data is marshaled as it flows across process boundaries. + +## Table of Contents + +- [Overview](#overview) +- [Key Components](#key-components) +- [Outbound Flow: Host Application to Kernel](#outbound-flow-host-application-to-kernel) +- [Inbound Flow: Kernel to Host Application](#inbound-flow-kernel-to-host-application) +- [Slot Types](#slot-types) +- [Custom Conversion Functions](#custom-conversion-functions) +- [Supported Data Types](#supported-data-types) + +## Overview + +The serialization pipeline enables communication between the host application (running in the main process) and the kernel (running in a web worker). This involves multiple levels of marshaling to handle object references across process boundaries. + +The pipeline uses three distinct marshals, each handling a different scope: + +1. **CapTP marshal** - Handles cross-process communication via `postMessage` +2. **Kernel marshal** - Handles kernel-internal message storage and vat delivery +3. **PresenceManager marshal** - Converts kernel references to callable presences for the host + +## Key Components + +### Source Files + +| Component | Location | Purpose | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| KRef-Presence utilities | [`packages/ocap-kernel/src/kref-presence.ts`](../packages/ocap-kernel/src/kref-presence.ts) | Converts between KRefs and presences | +| Kernel facade | [`packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.ts`](../packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.ts) | CapTP interface to kernel | +| KernelQueue | [`packages/ocap-kernel/src/KernelQueue.ts`](../packages/ocap-kernel/src/KernelQueue.ts) | Queues and processes messages | +| Kernel marshal | [`packages/ocap-kernel/src/liveslots/kernel-marshal.ts`](../packages/ocap-kernel/src/liveslots/kernel-marshal.ts) | Serializes data for kernel storage | + +### Marshals in the System + +| Marshal | Location | Slot Type | Body Format | When Used | +| ----------------------- | ------------------- | ------------ | ----------- | ---------------------------- | +| CapTP marshal | `@endo/captp` | `o+N`, `p+N` | capdata | Cross-process `E()` calls | +| Kernel marshal | `kernel-marshal.ts` | `ko*`, `kp*` | smallcaps | Kernel-to-vat messages | +| PresenceManager marshal | `kref-presence.ts` | `ko*`, `kp*` | smallcaps | Deserialize results for host | + +## Outbound Flow: Host Application to Kernel + +When the host application sends a message to a vat via the kernel: + +``` +Host Application Kernel Worker + │ │ + │ 1. Prepare call with kref strings │ + │ { target: 'ko42', method: 'foo' } │ + │ │ + │ 2. convertKrefsToStandins() │ + │ 'ko42' → kslot() → Exo remotable │ + │ │ + │ 3. E(kernelFacade).queueMessage() │ + │ CapTP serialize: remotable → o+1 │ + │ │ + │ ──────── postMessage channel ──────────► │ + │ │ + │ │ 4. CapTP deserialize + │ │ o+1 → remotable + │ │ + │ │ 5. kser([method, args]) + │ │ remotable → 'ko42' in slots + │ │ + │ │ 6. Message stored for vat delivery + │ │ Format: CapData + │ │ +``` + +### Step-by-Step Breakdown + +1. **Host prepares call** - The host application prepares a message with kref strings identifying the target object and any object references in the arguments. + +2. **Convert krefs to standins** - `convertKrefsToStandins()` transforms kref strings into Exo remotable objects that CapTP can serialize. This happens in `kernel-facade.ts`. + +3. **CapTP serializes** - When `E(kernelFacade).queueMessage()` is called, CapTP's internal marshal converts the remotable objects into CapTP-style slots (`o+1`, `p+2`, etc.). + +4. **CapTP deserializes** - On the kernel worker side, CapTP converts the slots back to remotable objects. + +5. **Kernel marshal serializes** - `kser([method, args])` converts the remotables to CapData with kref slots (`ko42`, `kp99`). + +6. **Message stored** - The kernel stores the message in `CapData` format for delivery to the target vat. + +## Inbound Flow: Kernel to Host Application + +When a vat returns a result back to the host application: + +``` +Kernel Worker Host Application + │ │ + │ 1. Vat executes and returns result │ + │ Format: CapData │ + │ │ + │ 2. Kernel resolves promise │ + │ CapData associated with kp │ + │ │ + │ 3. CapTP serializes result │ + │ CapTP message with CapData payload │ + │ │ + │ ◄─────── postMessage channel ───────── │ + │ │ + │ │ 4. CapTP delivers answer + │ │ Result: CapData + │ │ + │ │ 5. PresenceManager.fromCapData() + │ │ slots['ko42'] → makeKrefPresence() + │ │ + │ │ 6. Host receives E()-callable objects + │ │ +``` + +### Step-by-Step Breakdown + +1. **Vat returns result** - The vat executes the requested method and returns a result, which liveslots marshals into `CapData` format. + +2. **Kernel resolves promise** - The kernel associates the result with the kernel promise (`kp`) that represents the pending call. + +3. **CapTP serializes** - CapTP marshals the result (which contains `CapData`) for transport back to the host. + +4. **CapTP delivers answer** - The host receives the CapTP answer message containing the `CapData` result. + +5. **PresenceManager converts** - `PresenceManager.fromCapData()` deserializes the result, converting kref slots into `E()`-callable presence objects. + +6. **Host receives presences** - The host application receives JavaScript objects with presence objects that can be used with `E()` for further calls. + +## Slot Types + +The system uses two different slot naming schemes: + +### CapTP Slots + +Used by `@endo/captp` for cross-process object references: + +| Prefix | Meaning | +| ------ | ----------------------------------- | +| `o+N` | Exported object (positive = export) | +| `o-N` | Imported object (negative = import) | +| `p+N` | Exported promise | +| `p-N` | Imported promise | + +### Kernel Slots (KRefs) + +Used by the kernel for internal object tracking: + +| Prefix | Meaning | +| ------ | -------------- | +| `ko` | Kernel object | +| `kp` | Kernel promise | +| `kd` | Kernel device | +| `v` | Vat reference | + +KRefs are globally unique within a kernel and survive across process restarts. + +## Custom Conversion Functions + +### `convertKrefsToStandins` + +**Location:** `packages/ocap-kernel/src/kref-presence.ts` + +**Direction:** Outbound (host to kernel) + +**Purpose:** Transforms kref strings into `kslot()` Exo remotable objects that CapTP can serialize. + +```typescript +// Input +{ target: 'ko42', data: { ref: 'ko43' } } + +// Output +{ target: , data: { ref: } } +``` + +### `convertPresencesToStandins` + +**Location:** `packages/ocap-kernel/src/kref-presence.ts` + +**Direction:** Outbound (host to kernel) + +**Purpose:** Combines presence-to-kref and kref-to-standin conversions. Transforms presence objects directly into standins. + +### `PresenceManager.fromCapData` + +**Location:** `packages/ocap-kernel/src/kref-presence.ts` + +**Direction:** Inbound (kernel to host) + +**Purpose:** Deserializes `CapData` into JavaScript objects with `E()`-callable presences. + +```typescript +// Input: CapData +{ body: '{"@qclass":"slot","index":0}', slots: ['ko42'] } + +// Output + +``` + +## Supported Data Types + +The serialization pipeline supports JSON-compatible data types plus special object-capability types: + +### Supported + +- Primitives: `string`, `number`, `boolean`, `null`, `undefined` +- Collections: `Array`, plain `Object` +- Special: `BigInt`, `Symbol` (well-known only) +- OCap types: Remotable objects, Promises + +### Not Supported + +- `CopyTagged` objects (custom tagged data) - not supported in this pipeline +- Circular references +- Functions (except as part of Remotable objects) +- DOM objects, Buffers, or other platform-specific types + +### Important Notes + +1. All data passing through the pipeline must be JSON-serializable at its core +2. Object references are converted to slot strings and back, not passed directly +3. Promises are tracked by the kernel and resolved asynchronously +4. Remotable objects become presences that queue messages rather than invoking methods directly + +## Why Two Levels of Marshaling? + +``` +Host Process Kernel Worker + │ │ + │ CapTP marshal │ Kernel marshal + │ (o+/p+ slots) │ (ko/kp slots) + │ │ + └────────── postMessage ──────────────┘ + │ + JSON transport +``` + +The two-level marshaling serves distinct purposes: + +- **CapTP marshal**: Provides a general-purpose RPC mechanism for cross-process object passing. It knows nothing about kernel internals and uses its own slot numbering. + +- **Kernel marshal**: Handles kernel-specific concerns like persistent object identity, vat isolation, and garbage collection. KRefs must be stable across kernel restarts. + +The separation allows the kernel to use any transport mechanism (not just CapTP) while maintaining consistent internal object references. diff --git a/package.json b/package.json index 36505a5a8..b6b39b083 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,8 @@ "vite>sass>@parcel/watcher": false, "vitest>@vitest/browser>webdriverio>@wdio/utils>edgedriver": false, "vitest>@vitest/browser>webdriverio>@wdio/utils>geckodriver": false, - "vitest>@vitest/mocker>msw": false + "vitest>@vitest/mocker>msw": false, + "@ocap/cli>@metamask/kernel-shims>@libp2p/webrtc>@ipshipyard/node-datachannel": false } }, "resolutions": { diff --git a/packages/extension/package.json b/packages/extension/package.json index 17619735b..70adc7b52 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -49,6 +49,7 @@ "@metamask/kernel-ui": "workspace:^", "@metamask/kernel-utils": "workspace:^", "@metamask/logger": "workspace:^", + "@metamask/ocap-kernel": "workspace:^", "@metamask/streams": "workspace:^", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/packages/extension/src/background.ts b/packages/extension/src/background.ts index c7bdb855a..01f3fecb5 100644 --- a/packages/extension/src/background.ts +++ b/packages/extension/src/background.ts @@ -5,14 +5,12 @@ import { isCapTPNotification, getCapTPMessage, } from '@metamask/kernel-browser-runtime'; -import type { - KernelFacade, - CapTPMessage, -} from '@metamask/kernel-browser-runtime'; +import type { CapTPMessage } from '@metamask/kernel-browser-runtime'; import defaultSubcluster from '@metamask/kernel-browser-runtime/default-cluster'; import { delay, isJsonRpcMessage, stringify } from '@metamask/kernel-utils'; import type { JsonRpcMessage } from '@metamask/kernel-utils'; import { Logger } from '@metamask/logger'; +import { makePresenceManager } from '@metamask/ocap-kernel'; import { ChromeRuntimeDuplexStream } from '@metamask/streams/browser'; defineGlobals(); @@ -20,12 +18,11 @@ defineGlobals(); const OFFSCREEN_DOCUMENT_PATH = '/offscreen.html'; const logger = new Logger('background'); let bootPromise: Promise | null = null; -let kernelP: Promise; -let ping: () => Promise; // With this we can click the extension action button to wake up the service worker. chrome.action.onClicked.addListener(() => { - ping?.().catch(logger.error); + globalThis.kernel !== undefined && + E(globalThis.kernel).ping().catch(logger.error); }); // Install/update @@ -108,12 +105,12 @@ async function main(): Promise { }); // Get the kernel remote presence - kernelP = backgroundCapTP.getKernel(); + const kernelP = backgroundCapTP.getKernel(); + globalThis.kernel = kernelP; - ping = async () => { - const result = await E(kernelP).ping(); - logger.info(result); - }; + // Create presence manager for E() calls on vat objects + const presenceManager = makePresenceManager({ kernel: kernelP }); + Object.assign(globalThis.captp, presenceManager); // Handle incoming CapTP messages from the kernel const drainPromise = offscreenStream.drain((message) => { @@ -127,8 +124,11 @@ async function main(): Promise { drainPromise.catch(logger.error); try { - await ping(); // Wait for the kernel to be ready - await startDefaultSubcluster(kernelP); + await E(kernelP).ping(); + const rootKref = await startDefaultSubcluster(); + if (rootKref) { + await greetBootstrapVat(rootKref); + } } catch (error) { offscreenStream.throw(error as Error).catch(logger.error); } @@ -147,19 +147,33 @@ async function main(): Promise { /** * Idempotently starts the default subcluster. * - * @param kernelPromise - Promise for the kernel facade. + * @returns The rootKref of the bootstrap vat if launched, undefined if subcluster already exists. */ -async function startDefaultSubcluster( - kernelPromise: Promise, -): Promise { - const status = await E(kernelPromise).getStatus(); +async function startDefaultSubcluster(): Promise { + const status = await E(globalThis.kernel).getStatus(); if (status.subclusters.length === 0) { - const result = await E(kernelPromise).launchSubcluster(defaultSubcluster); + const result = await E(globalThis.kernel).launchSubcluster( + defaultSubcluster, + ); logger.info(`Default subcluster launched: ${JSON.stringify(result)}`); - } else { - logger.info('Subclusters already exist. Not launching default subcluster.'); + return result.rootKref; } + logger.info('Subclusters already exist. Not launching default subcluster.'); + return undefined; +} + +/** + * Greets the bootstrap vat by calling its hello() method. + * + * @param rootKref - The kref of the bootstrap vat's root object. + */ +async function greetBootstrapVat(rootKref: string): Promise { + const rootPresence = captp.resolveKref(rootKref) as { + hello: (from: string) => string; + }; + const greeting = await E(rootPresence).hello('background'); + logger.info(`Got greeting from bootstrap vat: ${greeting}`); } /** @@ -169,19 +183,16 @@ function defineGlobals(): void { Object.defineProperty(globalThis, 'kernel', { configurable: false, enumerable: true, - writable: false, - value: {}, + writable: true, + value: undefined, }); - Object.defineProperties(globalThis.kernel, { - ping: { - get: () => ping, - }, - getKernel: { - value: async () => kernelP, - }, + Object.defineProperty(globalThis, 'captp', { + configurable: false, + enumerable: true, + writable: false, + value: {}, }); - harden(globalThis.kernel); Object.defineProperty(globalThis, 'E', { value: E, diff --git a/packages/extension/src/global.d.ts b/packages/extension/src/global.d.ts index 06dd91196..f1f7ba76e 100644 --- a/packages/extension/src/global.d.ts +++ b/packages/extension/src/global.d.ts @@ -1,4 +1,5 @@ import type { KernelFacade } from '@metamask/kernel-browser-runtime'; +import type { PresenceManager } from '@metamask/ocap-kernel'; // Type declarations for kernel dev console API. declare global { @@ -16,24 +17,19 @@ declare global { var E: typeof import('@endo/eventual-send').E; // eslint-disable-next-line no-var - var kernel: { - /** - * Ping the kernel to verify connectivity. - */ - ping: () => Promise; + var kernel: KernelFacade | Promise; - /** - * Get the kernel remote presence for use with E(). - * - * @returns A promise for the kernel facade remote presence. - * @example - * ```typescript - * const kernel = await kernel.getKernel(); - * const status = await E(kernel).getStatus(); - * ``` - */ - getKernel: () => Promise; - }; + /** + * CapTP utilities for resolving krefs to E()-callable presences. + * + * @example + * ```typescript + * const alice = captp.resolveKref('ko1'); + * await E(alice).hello('console'); + * ``` + */ + // eslint-disable-next-line no-var + var captp: PresenceManager; } export {}; diff --git a/packages/extension/test/e2e/object-registry.test.ts b/packages/extension/test/e2e/object-registry.test.ts index f54038a4a..b4fc02bed 100644 --- a/packages/extension/test/e2e/object-registry.test.ts +++ b/packages/extension/test/e2e/object-registry.test.ts @@ -73,7 +73,7 @@ test.describe('Object Registry', () => { await clearLogsButton.click(); await popupPage.click('button:text("Object Registry")'); await expect( - popupPage.locator('text=Alice (v1) - 5 objects, 4 promises'), + popupPage.locator('text=Alice (v1) - 5 objects, 5 promises'), ).toBeVisible(); const targetSelect = popupPage.locator('[data-testid="message-target"]'); await expect(targetSelect).toBeVisible(); @@ -102,7 +102,7 @@ test.describe('Object Registry', () => { await expect(messageResponse).toContainText('"body":"#\\"vat Alice got'); await expect(messageResponse).toContainText('"slots":['); await expect( - popupPage.locator('text=Alice (v1) - 5 objects, 6 promises'), + popupPage.locator('text=Alice (v1) - 5 objects, 7 promises'), ).toBeVisible(); }); diff --git a/packages/extension/test/e2e/persistence.test.ts b/packages/extension/test/e2e/persistence.test.ts index 916f65324..74ce4cc87 100644 --- a/packages/extension/test/e2e/persistence.test.ts +++ b/packages/extension/test/e2e/persistence.test.ts @@ -45,6 +45,8 @@ test.describe('Kernel Persistence', () => { await expect( newPopupPage.locator('text=Subcluster s2 - 1 Vat'), ).toBeVisible(); + // Wait for database to fully persist before reloading + await newPopupPage.waitForTimeout(1000); // reload the extension await newPopupPage.evaluate(() => chrome.runtime.reload()); await newPopupPage.close(); diff --git a/packages/kernel-browser-runtime/package.json b/packages/kernel-browser-runtime/package.json index 5ea33b466..e5d179a96 100644 --- a/packages/kernel-browser-runtime/package.json +++ b/packages/kernel-browser-runtime/package.json @@ -86,11 +86,11 @@ "devDependencies": { "@arethetypeswrong/cli": "^0.17.4", "@endo/eventual-send": "^1.3.4", + "@libp2p/webrtc": "5.2.24", "@metamask/auto-changelog": "^5.3.0", "@metamask/eslint-config": "^15.0.0", "@metamask/eslint-config-nodejs": "^15.0.0", "@metamask/eslint-config-typescript": "^15.0.0", - "@ocap/nodejs": "workspace:^", "@ocap/repo-tools": "workspace:^", "@ts-bridge/cli": "^0.6.3", "@ts-bridge/shims": "^0.1.1", diff --git a/packages/kernel-browser-runtime/src/index.ts b/packages/kernel-browser-runtime/src/index.ts index 4c10590e3..37786aa78 100644 --- a/packages/kernel-browser-runtime/src/index.ts +++ b/packages/kernel-browser-runtime/src/index.ts @@ -11,7 +11,7 @@ export * from './makeIframeVatWorker.ts'; export * from './PlatformServicesClient.ts'; export * from './PlatformServicesServer.ts'; export * from './utils/index.ts'; -export type { KernelFacade } from './types.ts'; +export type { KernelFacade, LaunchResult } from './types.ts'; export { makeBackgroundCapTP, isCapTPNotification, diff --git a/packages/kernel-browser-runtime/src/kernel-worker/captp/captp.integration.test.ts b/packages/kernel-browser-runtime/src/kernel-worker/captp/captp.integration.test.ts index 86dc2f942..0e3fe0cf0 100644 --- a/packages/kernel-browser-runtime/src/kernel-worker/captp/captp.integration.test.ts +++ b/packages/kernel-browser-runtime/src/kernel-worker/captp/captp.integration.test.ts @@ -1,6 +1,3 @@ -// Real endoify needed for CapTP and E() to work properly -import '@ocap/nodejs/endoify-ts'; - import { E } from '@endo/eventual-send'; import type { ClusterConfig, Kernel } from '@metamask/ocap-kernel'; import { describe, it, expect, vi, beforeEach } from 'vitest'; diff --git a/packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.test.ts b/packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.test.ts index cdaf77703..a27b54708 100644 --- a/packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.test.ts +++ b/packages/kernel-browser-runtime/src/kernel-worker/captp/kernel-facade.test.ts @@ -61,14 +61,11 @@ describe('makeKernelFacade', () => { }); it('returns result with subclusterId and rootKref from kernel', async () => { - const kernelResult = { + vi.mocked(mockKernel.launchSubcluster).mockResolvedValueOnce({ subclusterId: 's1', bootstrapRootKref: 'ko1', bootstrapResult: { body: '#null', slots: [] }, - }; - vi.mocked(mockKernel.launchSubcluster).mockResolvedValueOnce( - kernelResult, - ); + }); const config: ClusterConfig = makeClusterConfig(); @@ -123,6 +120,36 @@ describe('makeKernelFacade', () => { expect(mockKernel.queueMessage).toHaveBeenCalledTimes(1); }); + it('converts kref strings in args to standins', async () => { + const target: KRef = 'ko1'; + const method = 'sendTo'; + // Use ko refs only - kp refs become promise standins with different structure + const args = ['ko42', { target: 'ko99', data: 'hello' }]; + + await facade.queueMessage(target, method, args); + + // Verify the call was made + expect(mockKernel.queueMessage).toHaveBeenCalledTimes(1); + + // Get the actual args passed to kernel + const [, , processedArgs] = vi.mocked(mockKernel.queueMessage).mock + .calls[0]!; + + // First arg should be a standin with getKref method + const firstArg = processedArgs[0] as { getKref: () => string }; + expect(firstArg).toHaveProperty('getKref'); + expect(firstArg.getKref()).toBe('ko42'); + + // Second arg should be an object with converted kref + const secondArg = processedArgs[1] as { + target: { getKref: () => string }; + data: string; + }; + expect(secondArg.target).toHaveProperty('getKref'); + expect(secondArg.target.getKref()).toBe('ko99'); + expect(secondArg.data).toBe('hello'); + }); + it('returns result from kernel', async () => { const expectedResult = { body: '#{"answer":42}', slots: [] }; vi.mocked(mockKernel.queueMessage).mockResolvedValueOnce(expectedResult); diff --git a/packages/kernel-browser-runtime/vitest.integration.config.ts b/packages/kernel-browser-runtime/vitest.integration.config.ts index 01ea8c4b3..6c20f76c6 100644 --- a/packages/kernel-browser-runtime/vitest.integration.config.ts +++ b/packages/kernel-browser-runtime/vitest.integration.config.ts @@ -18,6 +18,11 @@ export default defineConfig((args) => { fileURLToPath( import.meta.resolve('@ocap/repo-tools/test-utils/fetch-mock'), ), + // Use endoify-node which imports @libp2p/webrtc before lockdown + // (webrtc imports reflect-metadata which modifies globalThis.Reflect) + fileURLToPath( + import.meta.resolve('@metamask/kernel-shims/endoify-node'), + ), ], }, }), diff --git a/packages/kernel-shims/package.json b/packages/kernel-shims/package.json index eed7d3e65..b83e446b6 100644 --- a/packages/kernel-shims/package.json +++ b/packages/kernel-shims/package.json @@ -22,6 +22,7 @@ "./endoify": "./dist/endoify.js", "./endoify-repair": "./dist/endoify-repair.js", "./eventual-send": "./dist/eventual-send.js", + "./endoify-node": "./src/endoify-node.js", "./package.json": "./package.json" }, "main": "./dist/endoify.js", @@ -53,6 +54,14 @@ "@endo/lockdown": "^1.0.18", "ses": "^1.14.0" }, + "peerDependencies": { + "@libp2p/webrtc": "^5.0.0" + }, + "peerDependenciesMeta": { + "@libp2p/webrtc": { + "optional": true + } + }, "devDependencies": { "@endo/bundle-source": "^4.1.2", "@metamask/auto-changelog": "^5.3.0", diff --git a/packages/kernel-shims/src/endoify-node.js b/packages/kernel-shims/src/endoify-node.js new file mode 100644 index 000000000..5707cbf49 --- /dev/null +++ b/packages/kernel-shims/src/endoify-node.js @@ -0,0 +1,13 @@ +/* global hardenIntrinsics */ + +// Node.js-specific endoify that imports modules which modify globals before lockdown. +// This file is NOT bundled - it must be imported directly from src/. + +import './endoify-repair.js'; + +// @libp2p/webrtc needs to modify globals in Node.js only, so we need to import +// it before hardening. +// eslint-disable-next-line import-x/no-unresolved -- peer dependency +import '@libp2p/webrtc'; + +hardenIntrinsics(); diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index c4acfa7e2..a1ade2d74 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -68,6 +68,7 @@ "@metamask/eslint-config": "^15.0.0", "@metamask/eslint-config-nodejs": "^15.0.0", "@metamask/eslint-config-typescript": "^15.0.0", + "@metamask/kernel-shims": "workspace:^", "@ocap/cli": "workspace:^", "@ocap/repo-tools": "workspace:^", "@typescript-eslint/eslint-plugin": "^8.29.0", diff --git a/packages/kernel-test/src/vatstore.test.ts b/packages/kernel-test/src/vatstore.test.ts index 991903cea..3b0a88775 100644 --- a/packages/kernel-test/src/vatstore.test.ts +++ b/packages/kernel-test/src/vatstore.test.ts @@ -1,4 +1,3 @@ -import '@ocap/nodejs/endoify-ts'; import type { VatStore, VatCheckpoint } from '@metamask/kernel-store'; import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import type { ClusterConfig } from '@metamask/ocap-kernel'; diff --git a/packages/kernel-test/vitest.config.ts b/packages/kernel-test/vitest.config.ts index 47cf711f6..964287570 100644 --- a/packages/kernel-test/vitest.config.ts +++ b/packages/kernel-test/vitest.config.ts @@ -12,7 +12,9 @@ export default defineConfig((args) => { test: { name: 'kernel-test', setupFiles: [ - fileURLToPath(import.meta.resolve('@ocap/nodejs/endoify-ts')), + fileURLToPath( + import.meta.resolve('@metamask/kernel-shims/endoify-node'), + ), ], testTimeout: 30_000, }, diff --git a/packages/nodejs-test-workers/package.json b/packages/nodejs-test-workers/package.json index 5f6c0d67d..1a9a60309 100644 --- a/packages/nodejs-test-workers/package.json +++ b/packages/nodejs-test-workers/package.json @@ -81,6 +81,7 @@ "node": "^20.11 || >=22" }, "dependencies": { + "@metamask/kernel-shims": "workspace:^", "@metamask/logger": "workspace:^", "@metamask/ocap-kernel": "workspace:^", "@ocap/nodejs": "workspace:^" diff --git a/packages/nodejs-test-workers/src/workers/mock-fetch.ts b/packages/nodejs-test-workers/src/workers/mock-fetch.ts index ccca51833..58afd4844 100644 --- a/packages/nodejs-test-workers/src/workers/mock-fetch.ts +++ b/packages/nodejs-test-workers/src/workers/mock-fetch.ts @@ -1,4 +1,4 @@ -import '@ocap/nodejs/endoify-mjs'; +import '@metamask/kernel-shims/endoify-node'; import { Logger } from '@metamask/logger'; import type { VatId } from '@metamask/ocap-kernel'; import { makeNodeJsVatSupervisor } from '@ocap/nodejs'; diff --git a/packages/nodejs/package.json b/packages/nodejs/package.json index a64b6713c..a159dae8f 100644 --- a/packages/nodejs/package.json +++ b/packages/nodejs/package.json @@ -23,8 +23,6 @@ "default": "./dist/index.cjs" } }, - "./endoify-mjs": "./dist/env/endoify.mjs", - "./endoify-ts": "./src/env/endoify.ts", "./package.json": "./package.json" }, "files": [ diff --git a/packages/nodejs/src/env/endoify.ts b/packages/nodejs/src/env/endoify.ts deleted file mode 100644 index e494bcb24..000000000 --- a/packages/nodejs/src/env/endoify.ts +++ /dev/null @@ -1,7 +0,0 @@ -import '@metamask/kernel-shims/endoify-repair'; - -// @libp2p/webrtc needs to modify globals in Node.js only, so we need to import -// it before hardening. -import '@libp2p/webrtc'; - -hardenIntrinsics(); diff --git a/packages/nodejs/src/index.ts b/packages/nodejs/src/index.ts index 6af1ec51b..e89eeb6c0 100644 --- a/packages/nodejs/src/index.ts +++ b/packages/nodejs/src/index.ts @@ -1,3 +1,10 @@ export { NodejsPlatformServices } from './kernel/PlatformServices.ts'; export { makeKernel } from './kernel/make-kernel.ts'; export { makeNodeJsVatSupervisor } from './vat/make-supervisor.ts'; + +// Re-export presence manager from ocap-kernel for E() support +export { makePresenceManager } from '@metamask/ocap-kernel'; +export type { + PresenceManager, + PresenceManagerOptions, +} from '@metamask/ocap-kernel'; diff --git a/packages/nodejs/src/kernel/PlatformServices.test.ts b/packages/nodejs/src/kernel/PlatformServices.test.ts index 0dce6ab9b..2c62e0f8f 100644 --- a/packages/nodejs/src/kernel/PlatformServices.test.ts +++ b/packages/nodejs/src/kernel/PlatformServices.test.ts @@ -1,5 +1,3 @@ -import '../env/endoify.ts'; - import { makeCounter } from '@metamask/kernel-utils'; import type { VatId } from '@metamask/ocap-kernel'; import { Worker as NodeWorker } from 'node:worker_threads'; diff --git a/packages/nodejs/src/kernel/make-kernel.test.ts b/packages/nodejs/src/kernel/make-kernel.test.ts index b54e57ef7..2fdfdb43d 100644 --- a/packages/nodejs/src/kernel/make-kernel.test.ts +++ b/packages/nodejs/src/kernel/make-kernel.test.ts @@ -1,5 +1,3 @@ -import '../env/endoify.ts'; - import { Kernel } from '@metamask/ocap-kernel'; import { describe, expect, it, vi } from 'vitest'; diff --git a/packages/nodejs/src/vat/vat-worker.test.ts b/packages/nodejs/src/vat/vat-worker.test.ts index 3df85e695..763215216 100644 --- a/packages/nodejs/src/vat/vat-worker.test.ts +++ b/packages/nodejs/src/vat/vat-worker.test.ts @@ -1,5 +1,3 @@ -import '../env/endoify.ts'; - import { makeCounter } from '@metamask/kernel-utils'; import type { VatId } from '@metamask/ocap-kernel'; import { makePromiseKitMock } from '@ocap/repo-tools/test-utils'; diff --git a/packages/nodejs/src/vat/vat-worker.ts b/packages/nodejs/src/vat/vat-worker.ts index 4eccdb196..c08d2f17d 100644 --- a/packages/nodejs/src/vat/vat-worker.ts +++ b/packages/nodejs/src/vat/vat-worker.ts @@ -1,4 +1,4 @@ -import '../env/endoify.ts'; +import '@metamask/kernel-shims/endoify-node'; import { Logger } from '@metamask/logger'; import type { VatId } from '@metamask/ocap-kernel'; diff --git a/packages/nodejs/test/e2e/PlatformServices.test.ts b/packages/nodejs/test/e2e/PlatformServices.test.ts index 2bd4fef41..14f444fb7 100644 --- a/packages/nodejs/test/e2e/PlatformServices.test.ts +++ b/packages/nodejs/test/e2e/PlatformServices.test.ts @@ -1,5 +1,3 @@ -import '../../src/env/endoify.ts'; - import { makeCounter } from '@metamask/kernel-utils'; import type { VatId } from '@metamask/ocap-kernel'; import { NodeWorkerDuplexStream } from '@metamask/streams'; diff --git a/packages/nodejs/test/e2e/kernel-to-host-captp.test.ts b/packages/nodejs/test/e2e/kernel-to-host-captp.test.ts new file mode 100644 index 000000000..8e6054266 --- /dev/null +++ b/packages/nodejs/test/e2e/kernel-to-host-captp.test.ts @@ -0,0 +1,311 @@ +import { E } from '@endo/eventual-send'; +import type { Kernel, ClusterConfig, KRef, VatId } from '@metamask/ocap-kernel'; +import { makePresenceManager } from '@metamask/ocap-kernel'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + +import { makeKernel } from '../../src/kernel/make-kernel.ts'; + +type Alice = { + performHandoff: ( + bob: Bob, + carol: Carol, + greeting: string, + name: string, + ) => Promise; +}; +type Bob = { + makeGreeter: (greeting: string) => Promise; +}; +type Carol = { + receiveAndGreet: (greeter: Greeter, name: string) => Promise; + storeExo: (exo: unknown) => Promise; + useStoredExo: (name: string) => Promise; +}; +type Greeter = { + greet: (name: string) => Promise; +}; + +type PromiseVat = { + makeGreeter: (greeting: string) => Promise; + makeDeferredPromise: () => Promise; + resolveDeferredPromise: (value: unknown) => void; + rejectDeferredPromise: (reason: string) => void; + getRejectingPromise: (reason: string) => Promise; + awaitPromiseArg: (promiseArg: Promise) => Promise; + awaitDeferredFromVat: (promiserVat: PromiseVat) => Promise; +}; + +/** + * Creates a map from vat names to their root krefs for a given subcluster. + * + * @param kernel - The kernel instance. + * @param subclusterId - The subcluster ID. + * @returns A record mapping vat names to their root krefs. + */ +function getVatRootKrefs( + kernel: Kernel, + subclusterId: string, +): Record { + const subcluster = kernel.getSubcluster(subclusterId); + if (!subcluster) { + throw new Error(`Subcluster ${subclusterId} not found`); + } + + const vatNames = Object.keys(subcluster.config.vats); + const vatIds: VatId[] = subcluster.vats; + + const result: Record = {}; + for (let i = 0; i < vatNames.length; i++) { + const vatName = vatNames[i]; + assert(vatName, `Vat name is undefined`); + const vatId = vatIds[i]; + assert(vatId, `Vat ID for ${vatName} is undefined`); + result[vatName] = kernel.pinVatRoot(vatId); + } + return result; +} + +describe('third-party handoff', { timeout: 15_000 }, () => { + let kernel: Kernel; + + beforeEach(async () => { + kernel = await makeKernel({}); + }); + + afterEach(async () => { + await kernel.clearStorage(); + }); + + it('alice passes exo from Bob to Carol (vat-internal handoff)', async () => { + // Launch subcluster with Alice, Bob, Carol + const config: ClusterConfig = { + bootstrap: 'alice', + vats: { + alice: { + bundleSpec: 'http://localhost:3000/alice-vat.bundle', + }, + bob: { + bundleSpec: 'http://localhost:3000/bob-vat.bundle', + }, + carol: { + bundleSpec: 'http://localhost:3000/carol-vat.bundle', + }, + }, + }; + + const { subclusterId, bootstrapRootKref } = + await kernel.launchSubcluster(config); + + // Create presence manager for E() calls + const presenceManager = makePresenceManager({ kernel }); + + // Get presences for each vat root + const alice = presenceManager.resolveKref(bootstrapRootKref) as Alice; + + // Get Bob and Carol krefs using the subcluster + const vatRootKrefs = getVatRootKrefs(kernel, subclusterId); + const bob = presenceManager.resolveKref(vatRootKrefs.bob as string) as Bob; + const carol = presenceManager.resolveKref( + vatRootKrefs.carol as string, + ) as Carol; + + // Test: Alice orchestrates the third-party handoff + // Alice calls Bob to get a greeter, then passes it to Carol + const result = await E(alice).performHandoff(bob, carol, 'Hello', 'World'); + expect(result).toBe('Hello, World!'); + }); + + it('external orchestration of third-party handoff', async () => { + // Launch subcluster with Bob and Carol only + const config: ClusterConfig = { + bootstrap: 'bob', + vats: { + bob: { + bundleSpec: 'http://localhost:3000/bob-vat.bundle', + }, + carol: { + bundleSpec: 'http://localhost:3000/carol-vat.bundle', + }, + }, + }; + + const { subclusterId, bootstrapRootKref } = + await kernel.launchSubcluster(config); + + // Create presence manager for E() calls + const presenceManager = makePresenceManager({ kernel }); + + // Get presences + const bob = presenceManager.resolveKref(bootstrapRootKref) as Bob; + const vatRootKrefs = getVatRootKrefs(kernel, subclusterId); + const carol = presenceManager.resolveKref( + vatRootKrefs.carol as string, + ) as Carol; + + // Test: External code orchestrates the handoff + // 1. Get exo from Bob + const greeter = await E(bob).makeGreeter('Greetings'); + + // 2. Pass exo to Carol (third-party handoff) + const greeting = await E(carol).receiveAndGreet(greeter, 'Universe'); + expect(greeting).toBe('Greetings, Universe!'); + }); +}); + +describe('kernel promise handling', { timeout: 15_000 }, () => { + let kernel: Kernel; + + beforeEach(async () => { + kernel = await makeKernel({}); + }); + + afterEach(async () => { + await kernel.clearStorage(); + }); + + it('propagates promise rejection to host', async () => { + const config: ClusterConfig = { + bootstrap: 'promiseVat', + vats: { + promiseVat: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + }, + }; + + const { bootstrapRootKref } = await kernel.launchSubcluster(config); + const presenceManager = makePresenceManager({ kernel }); + const promiseVat = presenceManager.resolveKref( + bootstrapRootKref, + ) as PromiseVat; + + // Rejections from vats are delivered as Error objects + const result = await E(promiseVat).getRejectingPromise('test error'); + expect(result).toBeInstanceOf(Error); + expect((result as Error).message).toBe('test error'); + }); + + it('handles deferred promise resolution', async () => { + const config: ClusterConfig = { + bootstrap: 'promiseVat', + vats: { + promiseVat: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + }, + }; + + const { bootstrapRootKref } = await kernel.launchSubcluster(config); + const presenceManager = makePresenceManager({ kernel }); + const promiseVat = presenceManager.resolveKref( + bootstrapRootKref, + ) as PromiseVat; + + // Get a deferred promise (unresolved) + const deferredPromise = E(promiseVat).makeDeferredPromise(); + + // Resolve it + await E(promiseVat).resolveDeferredPromise('resolved value'); + + // The deferred promise should now resolve + const result = await deferredPromise; + expect(result).toBe('resolved value'); + }); + + it('handles deferred promise rejection', async () => { + const config: ClusterConfig = { + bootstrap: 'promiseVat', + vats: { + promiseVat: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + }, + }; + + const { bootstrapRootKref } = await kernel.launchSubcluster(config); + const presenceManager = makePresenceManager({ kernel }); + const promiseVat = presenceManager.resolveKref( + bootstrapRootKref, + ) as PromiseVat; + + // Get a deferred promise (unresolved) + const deferredPromise = E(promiseVat).makeDeferredPromise(); + + // Reject it + await E(promiseVat).rejectDeferredPromise('error reason'); + + // Rejections from vats are delivered as Error objects + const result = await deferredPromise; + expect(result).toBeInstanceOf(Error); + expect((result as Error).message).toBe('error reason'); + }); + + it('supports promise pipelining (E() on unresolved promise)', async () => { + const config: ClusterConfig = { + bootstrap: 'promiseVat', + vats: { + promiseVat: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + }, + }; + + const { bootstrapRootKref } = await kernel.launchSubcluster(config); + const presenceManager = makePresenceManager({ kernel }); + const promiseVat = presenceManager.resolveKref( + bootstrapRootKref, + ) as PromiseVat; + + // Get a promise for an exo (without awaiting) + const exoPromise = E(promiseVat).makeGreeter('Hi'); + + // Pipeline: call method on the unresolved promise + const greetingPromise = E(exoPromise).greet('World'); + + // Both should resolve correctly + const greeting = await greetingPromise; + expect(greeting).toBe('Hi, World!'); + }); + + it('passes deferred promise from one vat to another (cross-vat handoff)', async () => { + // Launch subcluster with two promise-vats + const config: ClusterConfig = { + bootstrap: 'promiser', + vats: { + promiser: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + awaiter: { + bundleSpec: 'http://localhost:3000/promise-vat.bundle', + }, + }, + }; + + const { subclusterId, bootstrapRootKref } = + await kernel.launchSubcluster(config); + + const presenceManager = makePresenceManager({ kernel }); + + // Get presences for both vats + const promiser = presenceManager.resolveKref( + bootstrapRootKref, + ) as PromiseVat; + const vatRootKrefs = getVatRootKrefs(kernel, subclusterId); + const awaiter = presenceManager.resolveKref( + vatRootKrefs.awaiter as string, + ) as PromiseVat; + + // 1. Get deferred promise from promiser (creates kp kref) + const deferredPromise = E(promiser).makeDeferredPromise(); + + // 2. Pass the promise to awaiter (kernel should pass kp kref) + const awaiterResult = E(awaiter).awaitPromiseArg(deferredPromise); + + // 3. Resolve the deferred promise from promiser + await E(promiser).resolveDeferredPromise('cross-vat value'); + + // 4. awaiterResult should now resolve + const result = await awaiterResult; + expect(result).toBe('received: cross-vat value'); + }); +}); diff --git a/packages/nodejs/test/e2e/kernel-worker.test.ts b/packages/nodejs/test/e2e/kernel-worker.test.ts index ba61e57cc..7573bf33e 100644 --- a/packages/nodejs/test/e2e/kernel-worker.test.ts +++ b/packages/nodejs/test/e2e/kernel-worker.test.ts @@ -1,5 +1,3 @@ -import '../../src/env/endoify.ts'; - import { Kernel } from '@metamask/ocap-kernel'; import type { ClusterConfig } from '@metamask/ocap-kernel'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; diff --git a/packages/nodejs/test/e2e/remote-comms.test.ts b/packages/nodejs/test/e2e/remote-comms.test.ts index 971eb6f00..545cdea9e 100644 --- a/packages/nodejs/test/e2e/remote-comms.test.ts +++ b/packages/nodejs/test/e2e/remote-comms.test.ts @@ -1,5 +1,3 @@ -import '../../src/env/endoify.ts'; - import type { Libp2p } from '@libp2p/interface'; import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { Kernel, kunser, makeKernelStore } from '@metamask/ocap-kernel'; diff --git a/packages/nodejs/test/vats/alice-vat.js b/packages/nodejs/test/vats/alice-vat.js new file mode 100644 index 000000000..343cdefcf --- /dev/null +++ b/packages/nodejs/test/vats/alice-vat.js @@ -0,0 +1,42 @@ +import { E } from '@endo/eventual-send'; +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Build function for Alice's vat. + * Alice orchestrates the third-party handoff between Bob and Carol. + * + * @param {object} vatPowers - Special powers granted to this vat. + * @param {object} vatPowers.logger - The logger object. + * @returns {object} The root object for the new vat. + */ +export function buildRootObject({ logger }) { + return makeDefaultExo('aliceRoot', { + bootstrap() { + logger.log('Alice vat bootstrap'); + }, + + /** + * Orchestrates a third-party handoff by getting an exo from Bob, + * passing it to Carol, and having Carol use it. + * + * @param {object} bob - Reference to Bob's vat root. + * @param {object} carol - Reference to Carol's vat root. + * @param {string} greeting - The greeting for Bob to use. + * @param {string} name - The name for Carol to greet. + * @returns {Promise} The greeting result. + */ + async performHandoff(bob, carol, greeting, name) { + logger.log('Alice starting handoff'); + + // Get exo from Bob + const greeter = await E(bob).makeGreeter(greeting); + logger.log('Alice received greeter from Bob'); + + // Pass to Carol and have her use it + const result = await E(carol).receiveAndGreet(greeter, name); + logger.log(`Alice got result: ${result}`); + + return result; + }, + }); +} diff --git a/packages/nodejs/test/vats/bob-vat.js b/packages/nodejs/test/vats/bob-vat.js new file mode 100644 index 000000000..25dc085c0 --- /dev/null +++ b/packages/nodejs/test/vats/bob-vat.js @@ -0,0 +1,34 @@ +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Build function for Bob's vat. + * Bob can create greeter exos that can be passed to other vats. + * + * @param {object} vatPowers - Special powers granted to this vat. + * @param {object} vatPowers.logger - The logger object. + * @returns {object} The root object for the new vat. + */ +export function buildRootObject({ logger }) { + return makeDefaultExo('bobRoot', { + bootstrap() { + logger.log('Bob vat bootstrap'); + }, + + /** + * Create a greeter exo that can greet with a custom message. + * This exo can be passed to other vats (third-party handoff). + * + * @param {string} greeting - The greeting prefix to use. + * @returns {object} A greeter exo with a greet method. + */ + makeGreeter(greeting) { + return makeDefaultExo('greeter', { + greet(name) { + const message = `${greeting}, ${name}!`; + logger.log(`Greeter says: ${message}`); + return message; + }, + }); + }, + }); +} diff --git a/packages/nodejs/test/vats/carol-vat.js b/packages/nodejs/test/vats/carol-vat.js new file mode 100644 index 000000000..b97244be7 --- /dev/null +++ b/packages/nodejs/test/vats/carol-vat.js @@ -0,0 +1,60 @@ +import { E } from '@endo/eventual-send'; +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Build function for Carol's vat. + * Carol can receive exos from other vats and call methods on them. + * + * @param {object} vatPowers - Special powers granted to this vat. + * @param {object} vatPowers.logger - The logger object. + * @returns {object} The root object for the new vat. + */ +export function buildRootObject({ logger }) { + /** @type {object | null} */ + let storedExo = null; + + return makeDefaultExo('carolRoot', { + bootstrap() { + logger.log('Carol vat bootstrap'); + }, + + /** + * Receive an exo and immediately call a method on it. + * This proves the third-party handoff worked. + * + * @param {object} exo - An exo received from another vat. + * @param {string} name - The name to greet. + * @returns {Promise} The greeting from the exo. + */ + receiveAndGreet(exo, name) { + logger.log(`Carol received exo and will greet "${name}"`); + return E(exo).greet(name); + }, + + /** + * Store an exo for later use. + * + * @param {object} exo - An exo to store. + * @returns {string} Confirmation message. + */ + storeExo(exo) { + storedExo = exo; + logger.log('Carol stored exo'); + return 'stored'; + }, + + /** + * Use a previously stored exo to greet. + * + * @param {string} name - The name to greet. + * @returns {Promise} The greeting from the stored exo. + */ + useStoredExo(name) { + if (!storedExo) { + throw new Error('No exo stored'); + } + logger.log(`Carol using stored exo to greet "${name}"`); + return E(storedExo).greet(name); + }, + }); +} diff --git a/packages/nodejs/test/vats/promise-vat.js b/packages/nodejs/test/vats/promise-vat.js new file mode 100644 index 000000000..223f376a4 --- /dev/null +++ b/packages/nodejs/test/vats/promise-vat.js @@ -0,0 +1,124 @@ +import { E } from '@endo/eventual-send'; +import { makeDefaultExo } from '@metamask/kernel-utils/exo'; + +/** + * Build function for a vat that tests promise behaviors. + * This vat provides methods to test kernel promise (kp kref) handling. + * + * @param {object} vatPowers - Special powers granted to this vat. + * @param {object} vatPowers.logger - The logger object. + * @returns {object} The root object for the new vat. + */ +export function buildRootObject({ logger }) { + let deferredResolver = null; + let deferredRejecter = null; + + return makeDefaultExo('promiseRoot', { + bootstrap() { + logger.log('Promise vat bootstrap'); + }, + + /** + * Returns a promise that resolves to a greeter exo. + * + * @param {string} greeting - The greeting prefix to use. + * @returns {Promise} A promise resolving to a greeter exo. + */ + makeGreeter(greeting) { + logger.log(`makeGreeter called with greeting: ${greeting}`); + return makeDefaultExo('greeter', { + greet(name) { + const message = `${greeting}, ${name}!`; + logger.log(`Greeter says: ${message}`); + return message; + }, + }); + }, + + /** + * Makes a deferred promise that can be resolved or rejected + * via separate method calls. + * + * @returns {Promise} An unresolved promise. + */ + makeDeferredPromise() { + logger.log('makeDeferredPromise called'); + return new Promise((resolve, reject) => { + deferredResolver = resolve; + deferredRejecter = reject; + }); + }, + + /** + * Resolves the deferred promise created by makeDeferredPromise. + * + * @param {unknown} value - The value to resolve with. + */ + resolveDeferredPromise(value) { + logger.log(`resolveDeferredPromise called with: ${value}`); + if (deferredResolver) { + deferredResolver(value); + deferredResolver = null; + deferredRejecter = null; + } else { + logger.log('No deferred promise to resolve'); + } + }, + + /** + * Rejects the deferred promise created by makeDeferredPromise. + * + * @param {string} reason - The rejection reason. + */ + rejectDeferredPromise(reason) { + logger.log(`rejectDeferredPromise called with reason: ${reason}`); + if (deferredRejecter) { + deferredRejecter(new Error(reason)); + deferredResolver = null; + deferredRejecter = null; + } else { + logger.log('No deferred promise to reject'); + } + }, + + /** + * Returns a promise that immediately rejects with the given reason. + * + * @param {string} reason - The rejection reason. + * @returns {Promise} A rejecting promise. + */ + getRejectingPromise(reason) { + logger.log(`getRejectingPromise called with reason: ${reason}`); + return Promise.reject(new Error(reason)); + }, + + /** + * Accepts a promise argument and awaits it before returning. + * + * @param {Promise} promiseArg - A promise to await. + * @returns {Promise} A message containing the resolved value. + */ + async awaitPromiseArg(promiseArg) { + logger.log('awaitPromiseArg called, awaiting promise...'); + const result = await promiseArg; + logger.log(`awaitPromiseArg resolved to: ${result}`); + return `received: ${result}`; + }, + + /** + * Gets a deferred promise from another vat and awaits it. + * This tests cross-vat kernel promise handling. + * + * @param {object} promiserVat - A reference to another promise-vat. + * @returns {Promise} A message containing the resolved value. + */ + async awaitDeferredFromVat(promiserVat) { + logger.log('awaitDeferredFromVat called, getting deferred promise...'); + const deferredPromise = E(promiserVat).makeDeferredPromise(); + logger.log('Got deferred promise, awaiting...'); + const result = await deferredPromise; + logger.log(`Deferred promise resolved to: ${result}`); + return `received: ${result}`; + }, + }); +} diff --git a/packages/nodejs/test/workers/stream-sync.js b/packages/nodejs/test/workers/stream-sync.js index 9b39391ad..0889812ea 100644 --- a/packages/nodejs/test/workers/stream-sync.js +++ b/packages/nodejs/test/workers/stream-sync.js @@ -1,4 +1,4 @@ -import '../../dist/env/endoify.mjs'; +import '@metamask/kernel-shims/endoify-node'; import { makeStreams } from '../../dist/vat/streams.mjs'; main().catch(console.error); diff --git a/packages/nodejs/vitest.config.e2e.ts b/packages/nodejs/vitest.config.e2e.ts index 3d803d822..922508bce 100644 --- a/packages/nodejs/vitest.config.e2e.ts +++ b/packages/nodejs/vitest.config.e2e.ts @@ -1,4 +1,5 @@ import { mergeConfig } from '@ocap/repo-tools/vitest-config'; +import { fileURLToPath } from 'node:url'; import { defineConfig, defineProject } from 'vitest/config'; import defaultConfig from '../../vitest.config.ts'; @@ -11,6 +12,11 @@ export default defineConfig((args) => { test: { name: 'nodejs:e2e', pool: 'forks', + setupFiles: [ + fileURLToPath( + import.meta.resolve('@metamask/kernel-shims/endoify-node'), + ), + ], include: ['./test/e2e/**/*.test.ts'], exclude: ['./src/**/*'], hookTimeout: 30_000, // Increase hook timeout for network cleanup diff --git a/packages/nodejs/vitest.config.ts b/packages/nodejs/vitest.config.ts index 0b8767bab..208d6346b 100644 --- a/packages/nodejs/vitest.config.ts +++ b/packages/nodejs/vitest.config.ts @@ -1,4 +1,5 @@ import { mergeConfig } from '@ocap/repo-tools/vitest-config'; +import { fileURLToPath } from 'node:url'; import { defineConfig, defineProject } from 'vitest/config'; import defaultConfig from '../../vitest.config.ts'; @@ -10,6 +11,11 @@ export default defineConfig((args) => { defineProject({ test: { name: 'nodejs', + setupFiles: [ + fileURLToPath( + import.meta.resolve('@metamask/kernel-shims/endoify-node'), + ), + ], include: ['./src/**/*.test.ts'], exclude: ['./test/e2e/'], }, diff --git a/packages/ocap-kernel/package.json b/packages/ocap-kernel/package.json index 10f251a7b..044301ac2 100644 --- a/packages/ocap-kernel/package.json +++ b/packages/ocap-kernel/package.json @@ -72,6 +72,7 @@ "@chainsafe/libp2p-noise": "^16.1.3", "@chainsafe/libp2p-yamux": "patch:@chainsafe/libp2p-yamux@npm%3A7.0.4#~/.yarn/patches/@chainsafe-libp2p-yamux-npm-7.0.4-284c2f6812.patch", "@endo/errors": "^1.2.13", + "@endo/eventual-send": "^1.3.4", "@endo/import-bundle": "^1.5.2", "@endo/marshal": "^1.8.0", "@endo/pass-style": "^1.6.3", diff --git a/packages/ocap-kernel/src/index.test.ts b/packages/ocap-kernel/src/index.test.ts index c7d68c1f4..70eb8de01 100644 --- a/packages/ocap-kernel/src/index.test.ts +++ b/packages/ocap-kernel/src/index.test.ts @@ -14,6 +14,7 @@ describe('index', () => { 'VatHandle', 'VatIdStruct', 'VatSupervisor', + 'convertKrefsToStandins', 'initTransport', 'isVatConfig', 'isVatId', @@ -22,6 +23,7 @@ describe('index', () => { 'kslot', 'kunser', 'makeKernelStore', + 'makePresenceManager', 'parseRef', ]); }); diff --git a/packages/ocap-kernel/src/index.ts b/packages/ocap-kernel/src/index.ts index 054a7b9db..ecba586e1 100644 --- a/packages/ocap-kernel/src/index.ts +++ b/packages/ocap-kernel/src/index.ts @@ -36,3 +36,12 @@ export type { SlotValue } from './liveslots/kernel-marshal.ts'; export { makeKernelStore } from './store/index.ts'; export type { KernelStore } from './store/index.ts'; export { parseRef } from './store/utils/parse-ref.ts'; +export { + makePresenceManager, + convertKrefsToStandins, +} from './kref-presence.ts'; +export type { + PresenceManager, + PresenceManagerOptions, + KernelLike, +} from './kref-presence.ts'; diff --git a/packages/ocap-kernel/src/kref-presence.test.ts b/packages/ocap-kernel/src/kref-presence.test.ts new file mode 100644 index 000000000..abc94ceb2 --- /dev/null +++ b/packages/ocap-kernel/src/kref-presence.test.ts @@ -0,0 +1,295 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +import type { PresenceManager, KernelLike } from './kref-presence.ts'; +import { makePresenceManager } from './kref-presence.ts'; +import { kslot } from './liveslots/kernel-marshal.ts'; + +// EHandler type definition (copied to avoid import issues with mocking) +type EHandler = { + get?: (target: object, prop: PropertyKey) => Promise; + applyMethod?: ( + target: object, + prop: PropertyKey, + args: unknown[], + ) => Promise; + applyFunction?: (target: object, args: unknown[]) => Promise; +}; + +// Hoisted mock setup - these must be defined before vi.mock() is hoisted +const { MockHandledPromise, mockE } = vi.hoisted(() => { + /** + * Mock HandledPromise that supports resolveWithPresence. + */ + class MockHandledPromiseImpl extends Promise { + constructor( + executor: ( + resolve: (value: TResult | PromiseLike) => void, + reject: (reason?: unknown) => void, + resolveWithPresence: (handler: EHandler) => object, + ) => void, + _handler?: EHandler, + ) { + let presence: object | undefined; + + const resolveWithPresence = (handler: EHandler): object => { + // Create a simple presence object that can receive E() calls + presence = new Proxy( + {}, + { + get(_target, prop) { + if (prop === Symbol.toStringTag) { + return 'Alleged: VatObject'; + } + // Return a function that calls the handler + return async (...args: unknown[]) => { + if (typeof prop === 'string') { + return handler.applyMethod?.(presence!, prop, args); + } + return undefined; + }; + }, + }, + ); + return presence; + }; + + super((resolve, reject) => { + executor(resolve, reject, resolveWithPresence); + }); + } + } + + // Mock E() to intercept calls on presences + const mockEImpl = (target: object) => { + return new Proxy( + {}, + { + get(_proxyTarget, prop) { + if (typeof prop === 'string') { + // Return a function that, when called, invokes the presence's method + return (...args: unknown[]) => { + const method = (target as Record)[prop]; + if (typeof method === 'function') { + return (method as (...a: unknown[]) => unknown)(...args); + } + // Try to get it from the proxy + return (target as Record unknown>)[ + prop + ]?.(...args); + }; + } + return undefined; + }, + }, + ); + }; + + return { + MockHandledPromise: MockHandledPromiseImpl, + mockE: mockEImpl, + }; +}); + +// Apply mocks +vi.mock('@endo/eventual-send', () => ({ + E: mockE, + HandledPromise: MockHandledPromise, +})); + +describe('makePresenceManager', () => { + let mockKernelLike: KernelLike; + let presenceManager: PresenceManager; + + beforeEach(() => { + mockKernelLike = { + ping: vi.fn(), + launchSubcluster: vi.fn(), + terminateSubcluster: vi.fn(), + queueMessage: vi.fn(), + getStatus: vi.fn(), + pingVat: vi.fn(), + getVatRoot: vi.fn(), + } as unknown as KernelLike; + + presenceManager = makePresenceManager({ + kernel: mockKernelLike, + }); + }); + + describe('resolveKref', () => { + it('returns a presence object for a kref', () => { + const presence = presenceManager.resolveKref('ko42'); + + expect(presence).toBeDefined(); + expect(typeof presence).toBe('object'); + }); + + it('returns the same presence for the same kref (memoization)', () => { + const presence1 = presenceManager.resolveKref('ko42'); + const presence2 = presenceManager.resolveKref('ko42'); + + expect(presence1).toBe(presence2); + }); + + it('returns different presences for different krefs', () => { + const presence1 = presenceManager.resolveKref('ko1'); + const presence2 = presenceManager.resolveKref('ko2'); + + expect(presence1).not.toBe(presence2); + }); + }); + + describe('krefOf', () => { + it('returns the kref for a known presence', () => { + const presence = presenceManager.resolveKref('ko42'); + const kref = presenceManager.krefOf(presence); + + expect(kref).toBe('ko42'); + }); + + it('returns undefined for an unknown object', () => { + const unknownObject = { foo: 'bar' }; + const kref = presenceManager.krefOf(unknownObject); + + expect(kref).toBeUndefined(); + }); + }); + + describe('presence-to-standin conversion in sendToKernel', () => { + // These tests verify that presences are recursively converted to standin + // objects (via kslot) when passed as arguments to E() calls on presences. + // The kernel's queueMessage expects standin objects, not presences. + + beforeEach(() => { + // Set up queueMessage to return a valid CapData response + vi.mocked(mockKernelLike.queueMessage).mockResolvedValue({ + body: '#null', + slots: [], + }); + }); + + it('converts top-level presence argument to standin', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (arg: unknown) => unknown; + }; + const argPresence = presenceManager.resolveKref('ko2'); + + // Call method with presence as argument + await targetPresence.someMethod(argPresence); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [kslot('ko2')], + ); + }); + + it('converts nested presence in object argument to standin', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (arg: { nested: unknown }) => unknown; + }; + const nestedPresence = presenceManager.resolveKref('ko2'); + + await targetPresence.someMethod({ nested: nestedPresence }); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [{ nested: kslot('ko2') }], + ); + }); + + it('converts presences in array argument to standins', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (arg: unknown[]) => unknown; + }; + const presence2 = presenceManager.resolveKref('ko2'); + const presence3 = presenceManager.resolveKref('ko3'); + + await targetPresence.someMethod([presence2, presence3]); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [[kslot('ko2'), kslot('ko3')]], + ); + }); + + it('converts deeply nested presences to standins', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (arg: { a: { b: { c: unknown } } }) => unknown; + }; + const deepPresence = presenceManager.resolveKref('ko99'); + + await targetPresence.someMethod({ a: { b: { c: deepPresence } } }); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [{ a: { b: { c: kslot('ko99') } } }], + ); + }); + + it('handles mixed arguments with primitives and nested presences', async () => { + type Args = [string, { nested: unknown }, unknown, number]; + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (...args: Args) => unknown; + }; + const presence2 = presenceManager.resolveKref('ko2'); + const presence3 = presenceManager.resolveKref('ko3'); + + await targetPresence.someMethod( + 'primitive', + { nested: presence2 }, + presence3, + 42, + ); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + ['primitive', { nested: kslot('ko2') }, kslot('ko3'), 42], + ); + }); + + it('preserves non-presence objects unchanged', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: (arg: { data: string; count: number }) => unknown; + }; + + await targetPresence.someMethod({ data: 'value', count: 123 }); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [{ data: 'value', count: 123 }], + ); + }); + + it('handles array with mixed presences and primitives', async () => { + const targetPresence = presenceManager.resolveKref('ko1') as { + someMethod: ( + arg: [unknown, string, number, { key: unknown }], + ) => unknown; + }; + const presence2 = presenceManager.resolveKref('ko2'); + + await targetPresence.someMethod([ + presence2, + 'string', + 42, + { key: presence2 }, + ]); + + expect(mockKernelLike.queueMessage).toHaveBeenCalledWith( + 'ko1', + 'someMethod', + [[kslot('ko2'), 'string', 42, { key: kslot('ko2') }]], + ); + }); + }); + + // Note: fromCapData and full E() handler integration tests require the real + // Endo runtime environment with proper SES lockdown. These behaviors are + // tested in captp.integration.test.ts which runs with the real Endo setup. +}); diff --git a/packages/ocap-kernel/src/kref-presence.ts b/packages/ocap-kernel/src/kref-presence.ts new file mode 100644 index 000000000..a7c4b51b6 --- /dev/null +++ b/packages/ocap-kernel/src/kref-presence.ts @@ -0,0 +1,361 @@ +/** + * Presence manager for creating E()-usable presences from kernel krefs. + * + * This module provides "slot translation" - converting kernel krefs (ko*, kp*) + * into presences that can receive eventual sends via E(). Method calls on these + * presences are forwarded to kernel.queueMessage() through the existing CapTP + * connection. + */ +import { E, HandledPromise } from '@endo/eventual-send'; +import type { EHandler } from '@endo/eventual-send'; +import { makeMarshal, Remotable } from '@endo/marshal'; +import type { CapData } from '@endo/marshal'; + +import type { Kernel } from './Kernel.ts'; +import { isPromiseKRef, kslot } from './liveslots/kernel-marshal.ts'; +import type { KRef } from './types.ts'; + +type Methods = Record unknown>; + +/** + * Function type for sending messages to the kernel. + */ +type SendToKernelFn = ( + kref: string, + method: string, + args: unknown[], +) => Promise; + +/** + * Minimal interface for kernel-like objects that can queue messages. + * Both Kernel and KernelFacade (from kernel-browser-runtime) satisfy this. + */ +export type KernelLike = { + queueMessage: Kernel['queueMessage']; +}; + +/** + * Options for creating a presence manager. + */ +export type PresenceManagerOptions = { + /** + * A kernel or kernel facade that can queue messages. + * Can be a promise since E() works with promises. + */ + kernel: KernelLike | Promise; +}; + +/** + * The presence manager interface. + */ +export type PresenceManager = { + /** + * Resolve a kref string to an E()-usable presence or tracked promise. + * + * For object refs (ko*): Returns a presence that can receive E() calls. + * For promise refs (p*, kp*, rp*): Returns a tracked Promise. + * + * @param kref - The kernel reference string (e.g., 'ko42', 'kp123'). + * @returns A presence or tracked promise. + */ + resolveKref: (kref: KRef) => Methods | Promise; + + /** + * Extract the kref from a presence or tracked promise. + * + * @param value - A presence or tracked promise created by resolveKref. + * @returns The kref string, or undefined if not a tracked value. + */ + krefOf: (value: object) => KRef | undefined; + + /** + * Deserialize a CapData result into presences/promises. + * + * @param data - The CapData to deserialize. + * @returns The deserialized value with krefs converted to presences/promises. + */ + fromCapData: (data: CapData) => unknown; +}; + +/** + * Create a remote kit for a kref, similar to CapTP's makeRemoteKit. + * Returns a settler that can create an E()-callable presence. + * + * @param kref - The kernel reference string. + * @param sendToKernel - Function to send messages to the kernel. + * @returns An object with a resolveWithPresence method. + */ +function makeKrefRemoteKit( + kref: string, + sendToKernel: SendToKernelFn, +): { resolveWithPresence: () => object } { + // Handler that intercepts E() calls on the presence + const handler: EHandler = { + async get(_target, prop) { + if (typeof prop !== 'string') { + return undefined; + } + // Property access: E(presence).prop returns a promise + return sendToKernel(kref, prop, []); + }, + async applyMethod(_target, prop, args) { + if (typeof prop !== 'string') { + throw new Error('Method name must be a string'); + } + // Method call: E(presence).method(args) + return sendToKernel(kref, prop, args); + }, + applyFunction(_target, _args) { + // Function call: E(presence)(args) - not supported for kref presences + throw new Error('Cannot call kref presence as a function'); + }, + }; + + let resolveWithPresenceFn: + | ((presenceHandler: EHandler) => object) + | undefined; + + // Create a HandledPromise to get access to resolveWithPresence + // We don't actually use the promise - we just need the resolver + // eslint-disable-next-line no-new, @typescript-eslint/no-floating-promises + new HandledPromise((_resolve, _reject, resolveWithPresence) => { + resolveWithPresenceFn = resolveWithPresence; + }, handler); + + return { + resolveWithPresence: () => { + if (!resolveWithPresenceFn) { + throw new Error('resolveWithPresence not initialized'); + } + return resolveWithPresenceFn(handler); + }, + }; +} + +/** + * Create an E()-usable presence for a kref. + * + * @param kref - The kernel reference string. + * @param iface - Interface name for the remotable. + * @param sendToKernel - Function to send messages to the kernel. + * @returns A presence that can receive E() calls. + */ +function makeKrefPresence( + kref: string, + iface: string, + sendToKernel: SendToKernelFn, +): Methods { + const kit = makeKrefRemoteKit(kref, sendToKernel); + // Wrap the presence in Remotable for proper pass-style + return Remotable(iface, undefined, kit.resolveWithPresence()) as Methods; +} + +/** + * Create a presence manager for E() on vat objects. + * + * This creates presences from kernel krefs that forward method calls + * to kernel.queueMessage() via the existing CapTP connection. + * + * @param options - Options including the kernel facade. + * @param options.kernel - The kernel instance or presence. + * @returns The presence manager. + */ +export function makePresenceManager({ + kernel, +}: PresenceManagerOptions): PresenceManager { + // State for kref↔presence mapping (for ko* object refs) + const krefToPresence = new Map(); + const presenceToKref = new WeakMap(); + + // State for kref↔promise mapping (for p*, kp*, rp* promise refs) + const krefToPromise = new Map>(); + const promiseToKref = new WeakMap(); + + // Forward declaration for sendToKernel + // eslint-disable-next-line prefer-const + let marshal: ReturnType>; + + /** + * Recursively convert presence/promise objects directly to kernel standins. + * + * This combines conversions in one pass: + * 1. Presences (ko* refs) → kref strings (via presenceToKref WeakMap lookup) + * 2. Tracked promises (kp* refs) → kref strings (via promiseToKref WeakMap lookup) + * 3. E() HandledPromises → await to get underlying tracked value + * 4. Kref strings → standins (via kslot) + * + * The kernel's queueMessage uses kser() which expects standin objects, + * not presences or raw kref strings. + * + * @param value - The value to convert. + * @returns The value with presences/promises converted to standins. + */ + const convertPresencesToStandins = async ( + value: unknown, + ): Promise => { + // If it's a Promise, await it to get the tracked value + // E() returns HandledPromises that wrap presences/tracked promises + if (value instanceof Promise) { + if (promiseToKref.has(value)) { + return kslot(promiseToKref.get(value) as KRef); + } + const resolved = await value; + return convertPresencesToStandins(resolved); + } + + // Check if it's a known presence or tracked promise - convert to standin + if (typeof value === 'object' && value !== null) { + // Check presence map (ko* refs) + const presenceKref = presenceToKref.get(value); + if (presenceKref !== undefined) { + return kslot(presenceKref); + } + // Check promise map (kp* refs) + const promiseKref = promiseToKref.get(value); + if (promiseKref !== undefined) { + return kslot(promiseKref); + } + // Recursively process arrays + if (Array.isArray(value)) { + return Promise.all(value.map(convertPresencesToStandins)); + } + // Recursively process plain objects + const entries = await Promise.all( + Object.entries(value).map(async ([key, val]) => [ + key, + await convertPresencesToStandins(val), + ]), + ); + return Object.fromEntries(entries); + } + // Return primitives as-is + return value; + }; + + /** + * Send a message to the kernel and deserialize the result. + * + * @param kref - The target kernel reference. + * @param method - The method name to call. + * @param args - Arguments to pass to the method. + * @returns The deserialized result from the kernel. + */ + const sendToKernel: SendToKernelFn = async ( + kref: KRef, + method: string, + args: unknown[], + ): Promise => { + // Convert presence/promise args to standins for kernel serialization + // Also awaits E() HandledPromises to get underlying tracked values + const convertedArgs = await Promise.all( + args.map(convertPresencesToStandins), + ); + + // Call kernel via existing CapTP + const result: CapData = await E(kernel).queueMessage( + kref, + method, + convertedArgs, + ); + + // Deserialize result (krefs become presences) + return marshal.fromCapData(result); + }; + + /** + * Convert a kref slot to a presence or tracked promise. + * + * For object refs (ko*): Creates an E()-callable presence. + * For promise refs (p*, kp*, rp*): Creates a tracked Promise tagged with the kref. + * + * @param kref - The kernel reference string. + * @param iface - Optional interface name for the presence. + * @returns A presence object or tracked promise. + */ + const convertSlotToVal = ( + kref: KRef, + iface?: string, + ): Methods | Promise => { + // Handle promise krefs (p*, kp*, rp*) - create tracked Promise + if (isPromiseKRef(kref)) { + let tracked = krefToPromise.get(kref); + if (!tracked) { + // Create a standin promise tagged with the kref (like kernel-marshal does) + const standinP = Promise.resolve(`${kref} stand in`); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + Object.defineProperty(standinP, Symbol.toStringTag, { + value: kref, + enumerable: false, + }); + tracked = harden(standinP); + krefToPromise.set(kref, tracked); + promiseToKref.set(tracked, kref); + } + return tracked; + } + + // Handle object krefs (ko*) - create presence + let presence = krefToPresence.get(kref); + if (!presence) { + presence = makeKrefPresence( + kref, + iface ?? 'Alleged: VatObject', + sendToKernel, + ); + krefToPresence.set(kref, presence); + presenceToKref.set(presence, kref); + } + return presence; + }; + + /** + * Convert a presence or tracked promise to a kref slot. + * This is called by the marshal for pass-by-presence objects. + * Throws if the object is not a known kref presence or tracked promise. + * + * @param val - The value to convert to a kref. + * @returns The kernel reference string. + */ + const convertValToSlot = (val: unknown): KRef => { + if (typeof val === 'object' && val !== null) { + // Check presence map (ko* refs) + const presenceKref = presenceToKref.get(val); + if (presenceKref !== undefined) { + return presenceKref; + } + // Check promise map (kp* refs) + const promiseKref = promiseToKref.get(val); + if (promiseKref !== undefined) { + return promiseKref; + } + } + throw new Error('Cannot serialize unknown remotable object'); + }; + + // Same options as kernel-marshal.ts + marshal = makeMarshal(convertValToSlot, convertSlotToVal, { + serializeBodyFormat: 'smallcaps', + errorTagging: 'off', + }); + + return harden({ + resolveKref: (kref: KRef): Methods | Promise => { + return convertSlotToVal(kref, 'Alleged: VatObject'); + }, + + krefOf: (value: object): KRef | undefined => { + // Check presence map (ko* refs) + const presenceKref = presenceToKref.get(value); + if (presenceKref !== undefined) { + return presenceKref; + } + // Check promise map (kp* refs) + return promiseToKref.get(value); + }, + + fromCapData: (data: CapData): unknown => { + return marshal.fromCapData(data); + }, + }); +} +harden(makePresenceManager); diff --git a/packages/ocap-kernel/src/liveslots/kernel-marshal.test.ts b/packages/ocap-kernel/src/liveslots/kernel-marshal.test.ts index 22ed75931..0850439ce 100644 --- a/packages/ocap-kernel/src/liveslots/kernel-marshal.test.ts +++ b/packages/ocap-kernel/src/liveslots/kernel-marshal.test.ts @@ -1,10 +1,30 @@ import { passStyleOf } from '@endo/marshal'; import { describe, it, expect } from 'vitest'; -import { kslot, krefOf, kser, kunser, makeError } from './kernel-marshal.ts'; +import { + kslot, + krefOf, + kser, + kunser, + makeError, + isPromiseKRef, +} from './kernel-marshal.ts'; import type { SlotValue } from './kernel-marshal.ts'; describe('kernel-marshal', () => { + describe('isPromiseKRef', () => { + it.each([ + ['p1', true], + ['kp1', true], + ['rp1', true], + ['ko1', false], + ['o+1', false], + ['o-1', false], + ])('returns $1 for $0', (ref, expected) => { + expect(isPromiseKRef(ref)).toBe(expected); + }); + }); + describe('kslot', () => { it('creates promise standin for promise refs', () => { const promiseRefs = ['p1', 'kp1', 'rp1']; diff --git a/packages/ocap-kernel/src/liveslots/kernel-marshal.ts b/packages/ocap-kernel/src/liveslots/kernel-marshal.ts index a023a9445..396adf977 100644 --- a/packages/ocap-kernel/src/liveslots/kernel-marshal.ts +++ b/packages/ocap-kernel/src/liveslots/kernel-marshal.ts @@ -55,6 +55,16 @@ function getStandinPromiseTag(promise: Promise): string { return kref; } +/** + * Check if a kref is a promise reference. + * Promise krefs can start with 'p', 'kp', or 'rp'. + * + * @param kref - The kernel reference string. + * @returns True if the kref is a promise reference. + */ +export const isPromiseKRef = (kref: string): boolean => + kref.startsWith('p') || kref.startsWith('kp') || kref.startsWith('rp'); + /** * Obtain a value serializable via `kser` for a given KRef. * @@ -71,7 +81,7 @@ export function kslot(kref: string, iface: string = 'undefined'): SlotValue { // eslint-disable-next-line no-param-reassign iface = iface.slice(9); } - if (kref.startsWith('p') || kref.startsWith('kp') || kref.startsWith('rp')) { + if (isPromiseKRef(kref)) { return makeStandinPromise(kref); } const standinObject = makeDefaultExo(iface, { diff --git a/packages/ocap-kernel/vitest.config.ts b/packages/ocap-kernel/vitest.config.ts index e049418f5..6264a93d4 100644 --- a/packages/ocap-kernel/vitest.config.ts +++ b/packages/ocap-kernel/vitest.config.ts @@ -12,9 +12,9 @@ export default defineConfig((args) => { test: { name: 'kernel', setupFiles: [ - // This is actually a circular dependency relationship, but it's fine because we're - // targeting the TypeScript source file and not listing @ocap/nodejs in package.json. - fileURLToPath(import.meta.resolve('@ocap/nodejs/endoify-ts')), + fileURLToPath( + import.meta.resolve('@metamask/kernel-shims/endoify-node'), + ), ], }, }), diff --git a/packages/omnium-gatherum/README.md b/packages/omnium-gatherum/README.md index 688955bae..10a89a722 100644 --- a/packages/omnium-gatherum/README.md +++ b/packages/omnium-gatherum/README.md @@ -10,6 +10,31 @@ or `npm install @ocap/omnium-gatherum` +## Usage + +### Installing and using the `echo` caplet + +After loading the extension, open the background console (chrome://extensions → Omnium → "Inspect views: service worker") and run the following: + +```javascript +// 1. Load the echo caplet manifest +const { manifest } = await omnium.caplet.load('echo'); + +// 2. Install the caplet +const installResult = await omnium.caplet.install(manifest); + +// 3. Get the caplet's root kref +const capletInfo = await omnium.caplet.get(installResult.capletId); +const rootKref = capletInfo.rootKref; + +// 4. Resolve the kref to an E()-usable presence +const echoRoot = omnium.resolveKref(rootKref); + +// 5. Call the echo method +const result = await E(echoRoot).echo('Hello, world!'); +console.log(result); // "echo: Hello, world!" +``` + ## Contributing This package is part of a monorepo. Instructions for contributing can be found in the [monorepo README](https://github.com/MetaMask/ocap-kernel#readme). diff --git a/packages/omnium-gatherum/src/background.ts b/packages/omnium-gatherum/src/background.ts index b00d3d5e1..9ac213abd 100644 --- a/packages/omnium-gatherum/src/background.ts +++ b/packages/omnium-gatherum/src/background.ts @@ -5,13 +5,12 @@ import { isCapTPNotification, getCapTPMessage, } from '@metamask/kernel-browser-runtime'; -import type { - CapTPMessage, - KernelFacade, -} from '@metamask/kernel-browser-runtime'; +import type { CapTPMessage } from '@metamask/kernel-browser-runtime'; import { delay, isJsonRpcMessage, stringify } from '@metamask/kernel-utils'; import type { JsonRpcMessage } from '@metamask/kernel-utils'; import { Logger } from '@metamask/logger'; +import type { PresenceManager } from '@metamask/ocap-kernel'; +import { makePresenceManager } from '@metamask/ocap-kernel'; import { ChromeRuntimeDuplexStream } from '@metamask/streams/browser'; import { initializeControllers } from './controllers/index.ts'; @@ -27,7 +26,8 @@ let bootPromise: Promise | null = null; // With this we can click the extension action button to wake up the service worker. chrome.action.onClicked.addListener(() => { - omnium.ping?.().catch(logger.error); + globalThis.kernel !== undefined && + E(globalThis.kernel).ping().catch(logger.error); }); // Install/update @@ -108,12 +108,11 @@ async function main(): Promise { }); const kernelP = backgroundCapTP.getKernel(); - globals.setKernelP(kernelP); + globalThis.kernel = kernelP; - globals.setPing(async (): Promise => { - const result = await E(kernelP).ping(); - logger.info(result); - }); + // Create presence manager for E() on vat objects + const presenceManager = makePresenceManager({ kernel: kernelP }); + globals.setPresenceManager(presenceManager); try { const controllers = await initializeControllers({ @@ -144,9 +143,8 @@ async function main(): Promise { } type GlobalSetters = { - setKernelP: (value: Promise) => void; - setPing: (value: () => Promise) => void; setCapletController: (value: CapletControllerFacet) => void; + setPresenceManager: (value: PresenceManager) => void; }; /** @@ -155,6 +153,9 @@ type GlobalSetters = { * @returns A device for setting the global values. */ function defineGlobals(): GlobalSetters { + let capletController: CapletControllerFacet; + let presenceManager: PresenceManager; + Object.defineProperty(globalThis, 'E', { configurable: false, enumerable: true, @@ -162,6 +163,13 @@ function defineGlobals(): GlobalSetters { value: E, }); + Object.defineProperty(globalThis, 'kernel', { + configurable: false, + enumerable: true, + writable: true, + value: undefined, + }); + Object.defineProperty(globalThis, 'omnium', { configurable: false, enumerable: true, @@ -169,10 +177,6 @@ function defineGlobals(): GlobalSetters { value: {}, }); - let kernelP: Promise; - let ping: (() => Promise) | undefined; - let capletController: CapletControllerFacet; - /** * Load a caplet's manifest and bundle by ID. * @@ -211,12 +215,6 @@ function defineGlobals(): GlobalSetters { }; Object.defineProperties(globalThis.omnium, { - ping: { - get: () => ping, - }, - getKernel: { - value: async () => kernelP, - }, caplet: { value: harden({ install: async (manifest: CapletManifest) => @@ -230,18 +228,21 @@ function defineGlobals(): GlobalSetters { E(capletController).getCapletRoot(capletId), }), }, + resolveKref: { + get: () => presenceManager.resolveKref, + }, + krefOf: { + get: () => presenceManager.krefOf, + }, }); harden(globalThis.omnium); return { - setKernelP: (value) => { - kernelP = value; - }, - setPing: (value) => { - ping = value; - }, setCapletController: (value) => { capletController = value; }, + setPresenceManager: (value) => { + presenceManager = value; + }, }; } diff --git a/packages/omnium-gatherum/src/caplets/echo/echo-caplet.js b/packages/omnium-gatherum/src/caplets/echo/echo-caplet.js index b32a80311..c0d0ee31c 100644 --- a/packages/omnium-gatherum/src/caplets/echo/echo-caplet.js +++ b/packages/omnium-gatherum/src/caplets/echo/echo-caplet.js @@ -48,7 +48,7 @@ export function buildRootObject(_vatPowers, _parameters, _baggage) { */ echo(message) { log('Echoing message:', message); - return `Echo: ${message}`; + return `echo: ${message}`; }, }); } diff --git a/packages/omnium-gatherum/src/global.d.ts b/packages/omnium-gatherum/src/global.d.ts index 1b4b60bb4..545ed5d14 100644 --- a/packages/omnium-gatherum/src/global.d.ts +++ b/packages/omnium-gatherum/src/global.d.ts @@ -22,24 +22,10 @@ declare global { var E: typeof import('@endo/eventual-send').E; // eslint-disable-next-line no-var - var omnium: { - /** - * Ping the kernel to verify connectivity. - */ - ping: () => Promise; - - /** - * Get the kernel remote presence for use with E(). - * - * @returns A promise for the kernel facade remote presence. - * @example - * ```typescript - * const kernel = await omnium.getKernel(); - * const status = await E(kernel).getStatus(); - * ``` - */ - getKernel: () => Promise; + var kernel: KernelFacade | Promise; + // eslint-disable-next-line no-var + var omnium: { /** * Caplet management API. */ diff --git a/yarn.lock b/yarn.lock index 3e8e2e1f9..36e1812ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2291,6 +2291,7 @@ __metadata: "@endo/captp": "npm:^4.4.8" "@endo/eventual-send": "npm:^1.3.4" "@endo/marshal": "npm:^1.8.0" + "@libp2p/webrtc": "npm:5.2.24" "@metamask/auto-changelog": "npm:^5.3.0" "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" @@ -2308,7 +2309,6 @@ __metadata: "@metamask/superstruct": "npm:^3.2.1" "@metamask/utils": "npm:^11.9.0" "@ocap/kernel-platforms": "workspace:^" - "@ocap/nodejs": "workspace:^" "@ocap/repo-tools": "workspace:^" "@ts-bridge/cli": "npm:^0.6.3" "@ts-bridge/shims": "npm:^0.1.1" @@ -2457,6 +2457,11 @@ __metadata: typescript-eslint: "npm:^8.29.0" vite: "npm:^7.3.0" vitest: "npm:^4.0.16" + peerDependencies: + "@libp2p/webrtc": ^5.0.0 + peerDependenciesMeta: + "@libp2p/webrtc": + optional: true languageName: unknown linkType: soft @@ -2700,6 +2705,7 @@ __metadata: "@chainsafe/libp2p-noise": "npm:^16.1.3" "@chainsafe/libp2p-yamux": "patch:@chainsafe/libp2p-yamux@npm%3A7.0.4#~/.yarn/patches/@chainsafe-libp2p-yamux-npm-7.0.4-284c2f6812.patch" "@endo/errors": "npm:^1.2.13" + "@endo/eventual-send": "npm:^1.3.4" "@endo/import-bundle": "npm:^1.5.2" "@endo/marshal": "npm:^1.8.0" "@endo/pass-style": "npm:^1.6.3" @@ -3470,6 +3476,7 @@ __metadata: "@metamask/kernel-ui": "workspace:^" "@metamask/kernel-utils": "workspace:^" "@metamask/logger": "workspace:^" + "@metamask/ocap-kernel": "workspace:^" "@metamask/streams": "workspace:^" "@ocap/cli": "workspace:^" "@ocap/kernel-test": "workspace:^" @@ -3733,6 +3740,7 @@ __metadata: "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" "@metamask/eslint-config-typescript": "npm:^15.0.0" + "@metamask/kernel-shims": "workspace:^" "@metamask/kernel-store": "workspace:^" "@metamask/kernel-utils": "workspace:^" "@metamask/logger": "workspace:^" @@ -3834,6 +3842,7 @@ __metadata: "@metamask/eslint-config": "npm:^15.0.0" "@metamask/eslint-config-nodejs": "npm:^15.0.0" "@metamask/eslint-config-typescript": "npm:^15.0.0" + "@metamask/kernel-shims": "workspace:^" "@metamask/logger": "workspace:^" "@metamask/ocap-kernel": "workspace:^" "@ocap/nodejs": "workspace:^"