--- a/lib/cjs/puppeteer/cdp/Browser.d.ts +++ b/lib/cjs/puppeteer/cdp/Browser.d.ts @@ -32,6 +32,7 @@ _disposeContext(contextId?: string): Promise; wsEndpoint(): string; newPage(): Promise; + _connection(): Connection; _createPageInContext(contextId?: string): Promise; installExtension(path: string): Promise; uninstallExtension(id: string): Promise; --- a/lib/cjs/puppeteer/cdp/Browser.js +++ b/lib/cjs/puppeteer/cdp/Browser.js @@ -175,6 +175,10 @@ async newPage() { return await this.#defaultContext.newPage(); } + // rebrowser-patches: expose browser CDP session + _connection() { + return this.#connection; + } async _createPageInContext(contextId) { const { targetId } = await this.#connection.send('Target.createTarget', { url: 'about:blank', --- a/lib/cjs/puppeteer/cdp/ExecutionContext.d.ts +++ b/lib/cjs/puppeteer/cdp/ExecutionContext.d.ts @@ -22,6 +22,7 @@ bindingcalled: Protocol.Runtime.BindingCalledEvent; }> implements Disposable { #private; + _frameId: any; constructor(client: CDPSession, contextPayload: Protocol.Runtime.ExecutionContextDescription, world: IsolatedWorld); get id(): number; get puppeteerUtil(): Promise>; @@ -116,6 +117,10 @@ * {@link ElementHandle | element handle}. */ evaluateHandle = EvaluateFunc>(pageFunction: Func | string, ...args: Params): Promise>>>; + clear(newId: any): void; + __re__getMainWorld({ client, frameId, isWorker }: any): Promise; + __re__getIsolatedWorld({ client, frameId, worldName }: any): Promise; + acquireContextId(tryCount?: number): Promise; [disposeSymbol](): void; } //# sourceMappingURL=ExecutionContext.d.ts.map \ No newline at end of file --- a/lib/cjs/puppeteer/cdp/ExecutionContext.js +++ b/lib/cjs/puppeteer/cdp/ExecutionContext.js @@ -86,6 +86,7 @@ #client; #world; #id; + _frameId; #name; #disposables = new disposable_js_1.DisposableStack(); constructor(client, contextPayload, world) { @@ -96,16 +97,22 @@ if (contextPayload.name) { this.#name = contextPayload.name; } + // rebrowser-patches: keep frameId to use later + if (contextPayload.auxData?.frameId) { + this._frameId = contextPayload.auxData?.frameId; + } const clientEmitter = this.#disposables.use(new EventEmitter_js_1.EventEmitter(this.#client)); clientEmitter.on('Runtime.bindingCalled', this.#onBindingCalled.bind(this)); - clientEmitter.on('Runtime.executionContextDestroyed', async (event) => { - if (event.executionContextId === this.#id) { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + clientEmitter.on('Runtime.executionContextDestroyed', async (event) => { + if (event.executionContextId === this.#id) { + this[disposable_js_1.disposeSymbol](); + } + }); + clientEmitter.on('Runtime.executionContextsCleared', async () => { this[disposable_js_1.disposeSymbol](); - } - }); - clientEmitter.on('Runtime.executionContextsCleared', async () => { - this[disposable_js_1.disposeSymbol](); - }); + }); + } clientEmitter.on('Runtime.consoleAPICalled', this.#onConsoleAPI.bind(this)); clientEmitter.on(CDPSession_js_1.CDPSessionEvent.Disconnected, () => { this[disposable_js_1.disposeSymbol](); @@ -328,7 +335,181 @@ async evaluateHandle(pageFunction, ...args) { return await this.#evaluate(false, pageFunction, ...args); } + // rebrowser-patches: alternative to dispose + clear(newId) { + this.#id = newId; + this.#bindings = new Map(); + this.#bindingsInstalled = false; + this.#puppeteerUtil = undefined; + } + async __re__getMainWorld({ client, frameId, isWorker = false }) { + let contextId; + // random name to make it harder to detect for any 3rd party script by watching window object and events + const randomName = [...Array(Math.floor(Math.random() * (10 + 1)) + 10)].map(() => Math.random().toString(36)[2]).join(''); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] binding name = ${randomName}`); + // add the binding + await client.send('Runtime.addBinding', { + name: randomName, + }); + // listen for 'Runtime.bindingCalled' event + const bindingCalledHandler = ({ name, payload, executionContextId }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][bindingCalledHandler]', { + name, + payload, + executionContextId + }); + if (contextId > 0) { + // already acquired the id + return; + } + if (name !== randomName) { + // ignore irrelevant bindings + return; + } + if (payload !== frameId) { + // ignore irrelevant frames + return; + } + contextId = executionContextId; + // remove this listener + client.off('Runtime.bindingCalled', bindingCalledHandler); + }; + client.on('Runtime.bindingCalled', bindingCalledHandler); + if (isWorker) { + // workers don't support `Page.addScriptToEvaluateOnNewDocument` and `Page.createIsolatedWorld`, but there are no iframes inside of them, so it's safe to just use Runtime.evaluate + await client.send('Runtime.evaluate', { + expression: `this['${randomName}']('${frameId}')`, + }); + } + else { + // we could call the binding right from `addScriptToEvaluateOnNewDocument`, but this way it will be called in all existing frames and it's hard to distinguish children from the parent + await client.send('Page.addScriptToEvaluateOnNewDocument', { + source: `document.addEventListener('${randomName}', (e) => self['${randomName}'](e.detail.frameId))`, + runImmediately: true, + }); + // create new isolated world for this frame + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + // use randomName for worldName to distinguish from normal utility world + worldName: randomName, + grantUniveralAccess: true, + }); + // emit event in the specific frame from the isolated world + await client.send('Runtime.evaluate', { + expression: `document.dispatchEvent(new CustomEvent('${randomName}', { detail: { frameId: '${frameId}' } }))`, + contextId: createIsolatedWorldResult.executionContextId, + }); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] result:`, { contextId }); + return contextId; + } + async __re__getIsolatedWorld({ client, frameId, worldName }) { + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + worldName, + grantUniveralAccess: true, + }); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getIsolatedWorld] result:`, createIsolatedWorldResult); + return createIsolatedWorldResult.executionContextId; + } + // rebrowser-patches: get context id if it's missing + async acquireContextId(tryCount = 1) { + if (this.#id > 0) { + return; + } + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] id = ${this.#id}, name = ${this.#name}, fixMode = ${fixMode}, tryCount = ${tryCount}`); + let contextId; + let tryAgain = true; + let errorMessage = 'N/A'; + if (fixMode === 'addBinding') { + try { + if (this.#id === -2) { + // isolated world + contextId = await this.__re__getIsolatedWorld({ + client: this.#client, + frameId: this._frameId, + worldName: this.#name, + }); + } + else { + // main world + contextId = await this.__re__getMainWorld({ + client: this.#client, + frameId: this._frameId, + isWorker: this.#id === -3, + }); + } + } + catch (error) { + process.env['REBROWSER_PATCHES_DEBUG'] && console.error('[rebrowser-patches][acquireContextId] error:', error); + errorMessage = error.message; + if (error instanceof Error) { + if (error.message.includes('No frame for given id found') || + error.message.includes('Target closed') || + error.message.includes('Session closed')) { + // target doesn't exist anymore, don't try again + tryAgain = false; + } + } + (0, util_js_1.debugError)(error); + } + } + else if (fixMode === 'alwaysIsolated') { + if (this.#id === -3) { + throw new Error('[rebrowser-patches] web workers are not supported in alwaysIsolated mode'); + } + contextId = await this.__re__getIsolatedWorld({ + client: this.#client, + frameId: this._frameId, + worldName: this.#name, + }); + } + else if (fixMode === 'enableDisable') { + const executionContextCreatedHandler = ({ context }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][executionContextCreated] this.#id = ${this.#id}, name = ${this.#name}, contextId = ${contextId}, event.context.id = ${context.id}`); + if (contextId > 0) { + // already acquired the id + return; + } + if (this.#id === -1) { + // main world + if (context.auxData && context.auxData['isDefault']) { + contextId = context.id; + } + } + else if (this.#id === -2) { + // utility world + if (this.#name === context.name) { + contextId = context.id; + } + } + else if (this.#id === -3) { + // web worker + contextId = context.id; + } + }; + this.#client.on('Runtime.executionContextCreated', executionContextCreatedHandler); + await this.#client.send('Runtime.enable'); + await this.#client.send('Runtime.disable'); + this.#client.off('Runtime.executionContextCreated', executionContextCreatedHandler); + } + if (!contextId) { + if (!tryAgain || tryCount >= 3) { + throw new Error(`[rebrowser-patches] acquireContextId failed (tryAgain = ${tryAgain}, tryCount = ${tryCount}), errorMessage: ${errorMessage}`); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] failed, try again (tryCount = ${tryCount})`); + return this.acquireContextId(tryCount + 1); + } + this.#id = contextId; + } async #evaluate(returnByValue, pageFunction, ...args) { + // rebrowser-patches: context id is missing, acquire it and try again + if (this.#id < 0) { + await this.acquireContextId(); + // @ts-ignore + return this.#evaluate(returnByValue, pageFunction, ...args); + } const sourceUrlComment = (0, util_js_1.getSourceUrlComment)((0, util_js_1.getSourcePuppeteerURLIfAvailable)(pageFunction)?.toString() ?? util_js_1.PuppeteerURL.INTERNAL_URL); if ((0, util_js_1.isString)(pageFunction)) { --- a/lib/cjs/puppeteer/cdp/FrameManager.js +++ b/lib/cjs/puppeteer/cdp/FrameManager.js @@ -154,6 +154,10 @@ this.#onFrameStoppedLoading(event.frameId); }); session.on('Runtime.executionContextCreated', async (event) => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } await this.#frameTreeHandled?.valueOrThrow(); this.#onExecutionContextCreated(event.context, session); }); @@ -178,9 +182,16 @@ this.#frameTreeHandled?.resolve(); }), client.send('Page.setLifecycleEventsEnabled', { enabled: true }), - client.send('Runtime.enable').then(() => { - return this.#createIsolatedWorld(client, util_js_1.UTILITY_WORLD_NAME); - }), + (() => { + // rebrowser-patches: skip Runtime.enable + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][FrameManager] initialize'); + return this.#createIsolatedWorld(client, util_js_1.UTILITY_WORLD_NAME); + } + return client.send('Runtime.enable').then(() => { + return this.#createIsolatedWorld(client, util_js_1.UTILITY_WORLD_NAME); + }); + })(), ...(frame ? Array.from(this.#scriptsToEvaluateOnNewDocument.values()) : []).map(script => { @@ -190,6 +201,26 @@ return frame?.addExposedFunctionBinding(binding); }), ]); + // rebrowser-patches: manually create main world context + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + this.frames() + .filter(frame => { + return frame.client === client; + }).map(frame => { + const world = frame.worlds[IsolatedWorlds_js_1.MAIN_WORLD]; + const contextPayload = { + id: -1, + name: '', + auxData: { + frameId: frame._id, + } + }; + const context = new ExecutionContext_js_1.ExecutionContext(frame.client, + // @ts-ignore + contextPayload, world); + world.setContext(context); + }); + } } catch (error) { this.#frameTreeHandled?.resolve(); @@ -358,6 +389,23 @@ } this._frameTree.addFrame(frame); } + // rebrowser-patches: we cannot fully dispose contexts as they won't be recreated as we don't have Runtime events, + // instead, just mark it all empty + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches] onFrameNavigated, navigationType = ${navigationType}, id = ${framePayload.id}, url = ${framePayload.url}`); + for (const worldSymbol of [IsolatedWorlds_js_1.MAIN_WORLD, IsolatedWorlds_js_1.PUPPETEER_WORLD]) { + // @ts-ignore + if (frame?.worlds[worldSymbol].context) { + // @ts-ignore + const frameOrWorker = frame.worlds[worldSymbol].environment; + if ('clearDocumentHandle' in frameOrWorker) { + frameOrWorker.clearDocumentHandle(); + } + // @ts-ignore + frame.worlds[worldSymbol].context?.clear(worldSymbol === IsolatedWorlds_js_1.MAIN_WORLD ? -1 : -2); + } + } + } frame = await this._frameTree.waitForFrame(frameId); frame._navigated(framePayload); this.emit(FrameManagerEvents_js_1.FrameManagerEvent.FrameNavigated, frame); @@ -385,6 +433,24 @@ worldName: name, grantUniveralAccess: true, }) + .then((createIsolatedWorldResult) => { + // rebrowser-patches: save created context id + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + return; + } + if (!createIsolatedWorldResult?.executionContextId) { + // probably "Target closed" error, just ignore it + return; + } + // @ts-ignore + this.#onExecutionContextCreated({ + id: createIsolatedWorldResult.executionContextId, + name, + auxData: { + frameId: frame._id, + } + }, frame.client); + }) .catch(util_js_1.debugError); })); this.#isolatedWorlds.add(key); --- a/lib/cjs/puppeteer/cdp/IsolatedWorld.d.ts +++ b/lib/cjs/puppeteer/cdp/IsolatedWorld.d.ts @@ -12,9 +12,9 @@ import type { TimeoutSettings } from '../common/TimeoutSettings.js'; import type { EvaluateFunc, HandleFor } from '../common/types.js'; import { disposeSymbol } from '../util/disposable.js'; -import type { ExecutionContext } from './ExecutionContext.js'; +import { ExecutionContext } from './ExecutionContext.js'; import type { CdpFrame } from './Frame.js'; -import type { MAIN_WORLD, PUPPETEER_WORLD } from './IsolatedWorlds.js'; +import { MAIN_WORLD, PUPPETEER_WORLD } from './IsolatedWorlds.js'; import type { CdpWebWorker } from './WebWorker.js'; /** * @internal --- a/lib/cjs/puppeteer/cdp/IsolatedWorld.js +++ b/lib/cjs/puppeteer/cdp/IsolatedWorld.js @@ -1,4 +1,5 @@ "use strict"; +//@ts-nocheck /** * @license * Copyright 2019 Google Inc. @@ -12,6 +13,8 @@ const util_js_1 = require("../common/util.js"); const disposable_js_1 = require("../util/disposable.js"); const ElementHandle_js_1 = require("./ElementHandle.js"); +const ExecutionContext_js_1 = require("./ExecutionContext.js"); +const IsolatedWorlds_js_1 = require("./IsolatedWorlds.js"); const JSHandle_js_1 = require("./JSHandle.js"); /** * @internal @@ -70,6 +73,21 @@ * Waits for the next context to be set on the isolated world. */ async #waitForExecutionContext() { + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + if (fixMode === 'addBinding') { + const isMainWorld = this.#frameOrWorker.worlds[IsolatedWorlds_js_1.MAIN_WORLD] === this; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][waitForExecutionContext] frameId = ${this.#frameOrWorker._id}, isMainWorld = ${isMainWorld}`); + const contextPayload = { + id: isMainWorld ? -1 : -2, + name: isMainWorld ? '' : util_js_1.UTILITY_WORLD_NAME, + auxData: { + frameId: this.#frameOrWorker._id, + } + }; + const context = new ExecutionContext_js_1.ExecutionContext(this.client, contextPayload, this); + this.setContext(context); + return context; + } const error = new Error('Execution context was destroyed'); const result = await (0, rxjs_js_1.firstValueFrom)((0, util_js_1.fromEmitterEvent)(this.#emitter, 'context').pipe((0, rxjs_js_1.raceWith)((0, util_js_1.fromEmitterEvent)(this.#emitter, 'disposed').pipe((0, rxjs_js_1.map)(() => { // The message has to match the CDP message expected by the WaitTask class. @@ -107,6 +125,8 @@ if (!context) { context = await this.#waitForExecutionContext(); } + // rebrowser-patches: make sure id is acquired + await context.acquireContextId(); const { object } = await this.client.send('DOM.resolveNode', { backendNodeId: backendNodeId, executionContextId: context.id, --- a/lib/cjs/puppeteer/cdp/WebWorker.js +++ b/lib/cjs/puppeteer/cdp/WebWorker.js @@ -24,6 +24,10 @@ this.#targetType = targetType; this.#world = new IsolatedWorld_js_1.IsolatedWorld(this, new TimeoutSettings_js_1.TimeoutSettings()); this.#client.once('Runtime.executionContextCreated', async (event) => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } this.#world.setContext(new ExecutionContext_js_1.ExecutionContext(client, event.context, this.#world)); }); this.#world.emitter.on('consoleapicalled', async (event) => { @@ -42,7 +46,22 @@ }); // This might fail if the target is closed before we receive all execution contexts. networkManager?.addClient(this.#client).catch(util_js_1.debugError); - this.#client.send('Runtime.enable').catch(util_js_1.debugError); + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // @ts-ignore + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][WebWorker] initialize', targetType, targetId, client._target(), client._target()._getTargetInfo()); + // rebrowser-patches: manually create context + const contextPayload = { + id: -3, + auxData: { + frameId: targetId, + } + }; + // @ts-ignore + this.#world.setContext(new ExecutionContext_js_1.ExecutionContext(client, contextPayload, this.#world)); + } + else { + this.#client.send('Runtime.enable').catch(util_js_1.debugError); + } } mainRealm() { return this.#world; --- a/lib/cjs/puppeteer/common/util.js +++ b/lib/cjs/puppeteer/common/util.js @@ -252,7 +252,10 @@ /** * @internal */ -exports.UTILITY_WORLD_NAME = '__puppeteer_utility_world__' + version_js_1.packageVersion; +exports.UTILITY_WORLD_NAME = +// rebrowser-patches: change utility world name +process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] !== '0' ? (process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] || 'util') : + '__puppeteer_utility_world__' + version_js_1.packageVersion; /** * @internal */ @@ -261,6 +264,10 @@ * @internal */ function getSourceUrlComment(url) { + // rebrowser-patches: change sourceUrl to generic script name + if (process.env['REBROWSER_PATCHES_SOURCE_URL'] !== '0') { + url = process.env['REBROWSER_PATCHES_SOURCE_URL'] || 'app.js'; + } return `//# sourceURL=${url}`; } /** --- a/lib/es5-iife/puppeteer-core-browser.js +++ b/lib/es5-iife/puppeteer-core-browser.js @@ -3561,7 +3561,9 @@ /** * @internal */ - const UTILITY_WORLD_NAME = '__puppeteer_utility_world__' + packageVersion; + const UTILITY_WORLD_NAME = + // rebrowser-patches: change utility world name + process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] !== '0' ? process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] || 'util' : '__puppeteer_utility_world__' + packageVersion; /** * @internal */ @@ -3570,6 +3572,10 @@ * @internal */ function getSourceUrlComment(url) { + // rebrowser-patches: change sourceUrl to generic script name + if (process.env['REBROWSER_PATCHES_SOURCE_URL'] !== '0') { + url = process.env['REBROWSER_PATCHES_SOURCE_URL'] || 'app.js'; + } return `//# sourceURL=${url}`; } /** @@ -15625,6 +15631,7 @@ _classPrivateFieldInitSpec(this, _client8, void 0); _classPrivateFieldInitSpec(this, _world3, void 0); _classPrivateFieldInitSpec(this, _id5, void 0); + _defineProperty(this, "_frameId", void 0); _classPrivateFieldInitSpec(this, _name3, void 0); _classPrivateFieldInitSpec(this, _disposables, new DisposableStack()); // Contains mapping from functions that should be bound to Puppeteer functions. @@ -15640,16 +15647,22 @@ if (contextPayload.name) { _classPrivateFieldSet(_name3, this, contextPayload.name); } + // rebrowser-patches: keep frameId to use later + if (contextPayload.auxData?.frameId) { + this._frameId = contextPayload.auxData?.frameId; + } const clientEmitter = _classPrivateFieldGet(_disposables, this).use(new EventEmitter(_classPrivateFieldGet(_client8, this))); clientEmitter.on('Runtime.bindingCalled', _assertClassBrand(_ExecutionContext_brand, this, _onBindingCalled).bind(this)); - clientEmitter.on('Runtime.executionContextDestroyed', async event => { - if (event.executionContextId === _classPrivateFieldGet(_id5, this)) { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + clientEmitter.on('Runtime.executionContextDestroyed', async event => { + if (event.executionContextId === _classPrivateFieldGet(_id5, this)) { + this[disposeSymbol](); + } + }); + clientEmitter.on('Runtime.executionContextsCleared', async () => { this[disposeSymbol](); - } - }); - clientEmitter.on('Runtime.executionContextsCleared', async () => { - this[disposeSymbol](); - }); + }); + } clientEmitter.on('Runtime.consoleAPICalled', _assertClassBrand(_ExecutionContext_brand, this, _onConsoleAPI).bind(this)); clientEmitter.on(exports.CDPSessionEvent.Disconnected, () => { this[disposeSymbol](); @@ -15771,6 +15784,181 @@ async evaluateHandle(pageFunction, ...args) { return await _assertClassBrand(_ExecutionContext_brand, this, _evaluate).call(this, false, pageFunction, ...args); } + // rebrowser-patches: alternative to dispose + clear(newId) { + _classPrivateFieldSet(_id5, this, newId); + _classPrivateFieldSet(_bindings, this, new Map()); + _classPrivateFieldSet(_bindingsInstalled, this, false); + _classPrivateFieldSet(_puppeteerUtil, this, undefined); + } + async __re__getMainWorld({ + client, + frameId, + isWorker = false + }) { + let contextId; + // random name to make it harder to detect for any 3rd party script by watching window object and events + const randomName = [...Array(Math.floor(Math.random() * (10 + 1)) + 10)].map(() => Math.random().toString(36)[2]).join(''); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] binding name = ${randomName}`); + // add the binding + await client.send('Runtime.addBinding', { + name: randomName + }); + // listen for 'Runtime.bindingCalled' event + const bindingCalledHandler = ({ + name, + payload, + executionContextId + }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][bindingCalledHandler]', { + name, + payload, + executionContextId + }); + if (contextId > 0) { + // already acquired the id + return; + } + if (name !== randomName) { + // ignore irrelevant bindings + return; + } + if (payload !== frameId) { + // ignore irrelevant frames + return; + } + contextId = executionContextId; + // remove this listener + client.off('Runtime.bindingCalled', bindingCalledHandler); + }; + client.on('Runtime.bindingCalled', bindingCalledHandler); + if (isWorker) { + // workers don't support `Page.addScriptToEvaluateOnNewDocument` and `Page.createIsolatedWorld`, but there are no iframes inside of them, so it's safe to just use Runtime.evaluate + await client.send('Runtime.evaluate', { + expression: `this['${randomName}']('${frameId}')` + }); + } else { + // we could call the binding right from `addScriptToEvaluateOnNewDocument`, but this way it will be called in all existing frames and it's hard to distinguish children from the parent + await client.send('Page.addScriptToEvaluateOnNewDocument', { + source: `document.addEventListener('${randomName}', (e) => self['${randomName}'](e.detail.frameId))`, + runImmediately: true + }); + // create new isolated world for this frame + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + // use randomName for worldName to distinguish from normal utility world + worldName: randomName, + grantUniveralAccess: true + }); + // emit event in the specific frame from the isolated world + await client.send('Runtime.evaluate', { + expression: `document.dispatchEvent(new CustomEvent('${randomName}', { detail: { frameId: '${frameId}' } }))`, + contextId: createIsolatedWorldResult.executionContextId + }); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] result:`, { + contextId + }); + return contextId; + } + async __re__getIsolatedWorld({ + client, + frameId, + worldName + }) { + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + worldName, + grantUniveralAccess: true + }); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getIsolatedWorld] result:`, createIsolatedWorldResult); + return createIsolatedWorldResult.executionContextId; + } + // rebrowser-patches: get context id if it's missing + async acquireContextId(tryCount = 1) { + if (_classPrivateFieldGet(_id5, this) > 0) { + return; + } + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] id = ${_classPrivateFieldGet(_id5, this)}, name = ${_classPrivateFieldGet(_name3, this)}, fixMode = ${fixMode}, tryCount = ${tryCount}`); + let contextId; + let tryAgain = true; + let errorMessage = 'N/A'; + if (fixMode === 'addBinding') { + try { + if (_classPrivateFieldGet(_id5, this) === -2) { + // isolated world + contextId = await this.__re__getIsolatedWorld({ + client: _classPrivateFieldGet(_client8, this), + frameId: this._frameId, + worldName: _classPrivateFieldGet(_name3, this) + }); + } else { + // main world + contextId = await this.__re__getMainWorld({ + client: _classPrivateFieldGet(_client8, this), + frameId: this._frameId, + isWorker: _classPrivateFieldGet(_id5, this) === -3 + }); + } + } catch (error) { + process.env['REBROWSER_PATCHES_DEBUG'] && console.error('[rebrowser-patches][acquireContextId] error:', error); + errorMessage = error.message; + if (error instanceof Error) { + if (error.message.includes('No frame for given id found') || error.message.includes('Target closed') || error.message.includes('Session closed')) { + // target doesn't exist anymore, don't try again + tryAgain = false; + } + } + debugError(error); + } + } else if (fixMode === 'alwaysIsolated') { + if (_classPrivateFieldGet(_id5, this) === -3) { + throw new Error('[rebrowser-patches] web workers are not supported in alwaysIsolated mode'); + } + contextId = await this.__re__getIsolatedWorld({ + client: _classPrivateFieldGet(_client8, this), + frameId: this._frameId, + worldName: _classPrivateFieldGet(_name3, this) + }); + } else if (fixMode === 'enableDisable') { + const executionContextCreatedHandler = ({ + context + }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][executionContextCreated] this.#id = ${_classPrivateFieldGet(_id5, this)}, name = ${_classPrivateFieldGet(_name3, this)}, contextId = ${contextId}, event.context.id = ${context.id}`); + if (contextId > 0) { + // already acquired the id + return; + } + if (_classPrivateFieldGet(_id5, this) === -1) { + // main world + if (context.auxData && context.auxData['isDefault']) { + contextId = context.id; + } + } else if (_classPrivateFieldGet(_id5, this) === -2) { + // utility world + if (_classPrivateFieldGet(_name3, this) === context.name) { + contextId = context.id; + } + } else if (_classPrivateFieldGet(_id5, this) === -3) { + // web worker + contextId = context.id; + } + }; + _classPrivateFieldGet(_client8, this).on('Runtime.executionContextCreated', executionContextCreatedHandler); + await _classPrivateFieldGet(_client8, this).send('Runtime.enable'); + await _classPrivateFieldGet(_client8, this).send('Runtime.disable'); + _classPrivateFieldGet(_client8, this).off('Runtime.executionContextCreated', executionContextCreatedHandler); + } + if (!contextId) { + if (!tryAgain || tryCount >= 3) { + throw new Error(`[rebrowser-patches] acquireContextId failed (tryAgain = ${tryAgain}, tryCount = ${tryCount}), errorMessage: ${errorMessage}`); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] failed, try again (tryCount = ${tryCount})`); + return this.acquireContextId(tryCount + 1); + } + _classPrivateFieldSet(_id5, this, contextId); + } [disposeSymbol]() { _classPrivateFieldGet(_disposables, this).dispose(); this.emit('disposed', undefined); @@ -15870,6 +16058,12 @@ } } async function _evaluate(returnByValue, pageFunction, ...args) { + // rebrowser-patches: context id is missing, acquire it and try again + if (_classPrivateFieldGet(_id5, this) < 0) { + await this.acquireContextId(); + // @ts-ignore + return _assertClassBrand(_ExecutionContext_brand, this, _evaluate).call(this, returnByValue, pageFunction, ...args); + } const sourceUrlComment = getSourceUrlComment(getSourcePuppeteerURLIfAvailable(pageFunction)?.toString() ?? PuppeteerURL.INTERNAL_URL); if (isString(pageFunction)) { const contextId = _classPrivateFieldGet(_id5, this); @@ -16036,6 +16230,27 @@ /** * @license + * Copyright 2022 Google Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + /** + * A unique key for {@link IsolatedWorldChart} to denote the default world. + * Execution contexts are automatically created in the default world. + * + * @internal + */ + const MAIN_WORLD = Symbol('mainWorld'); + /** + * A unique key for {@link IsolatedWorldChart} to denote the puppeteer world. + * This world contains all puppeteer-internal bindings/code. + * + * @internal + */ + const PUPPETEER_WORLD = Symbol('puppeteerWorld'); + + //@ts-nocheck + /** + * @license * Copyright 2019 Google Inc. * SPDX-License-Identifier: Apache-2.0 */ @@ -16109,6 +16324,8 @@ if (!context) { context = await _assertClassBrand(_IsolatedWorld_brand, this, _waitForExecutionContext).call(this); } + // rebrowser-patches: make sure id is acquired + await context.acquireContextId(); const { object } = await this.client.send('DOM.resolveNode', { @@ -16164,15 +16381,9 @@ /** * @license - * Copyright 2022 Google Inc. + * Copyright 2019 Google Inc. * SPDX-License-Identifier: Apache-2.0 */ - /** - * A unique key for {@link IsolatedWorldChart} to denote the default world. - * Execution contexts are automatically created in the default world. - * - * @internal - */ function _onContextDisposed() { _classPrivateFieldSet(_context, this, undefined); if ('clearDocumentHandle' in _classPrivateFieldGet(_frameOrWorker, this)) { @@ -16195,6 +16406,21 @@ * Waits for the next context to be set on the isolated world. */ async function _waitForExecutionContext() { + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + if (fixMode === 'addBinding') { + const isMainWorld = _classPrivateFieldGet(_frameOrWorker, this).worlds[MAIN_WORLD] === this; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][waitForExecutionContext] frameId = ${_classPrivateFieldGet(_frameOrWorker, this)._id}, isMainWorld = ${isMainWorld}`); + const contextPayload = { + id: isMainWorld ? -1 : -2, + name: isMainWorld ? '' : UTILITY_WORLD_NAME, + auxData: { + frameId: _classPrivateFieldGet(_frameOrWorker, this)._id + } + }; + const context = new ExecutionContext(this.client, contextPayload, this); + this.setContext(context); + return context; + } const error = new Error('Execution context was destroyed'); const result = await firstValueFrom(fromEmitterEvent(_classPrivateFieldGet(_emitter2, this), 'context').pipe(raceWith(fromEmitterEvent(_classPrivateFieldGet(_emitter2, this), 'disposed').pipe(map(() => { // The message has to match the CDP message expected by the WaitTask class. @@ -16202,20 +16428,6 @@ })), timeout(this.timeoutSettings.timeout())))); return result; } - const MAIN_WORLD = Symbol('mainWorld'); - /** - * A unique key for {@link IsolatedWorldChart} to denote the puppeteer world. - * This world contains all puppeteer-internal bindings/code. - * - * @internal - */ - const PUPPETEER_WORLD = Symbol('puppeteerWorld'); - - /** - * @license - * Copyright 2019 Google Inc. - * SPDX-License-Identifier: Apache-2.0 - */ const puppeteerToProtocolLifecycle = new Map([['load', 'load'], ['domcontentloaded', 'DOMContentLoaded'], ['networkidle0', 'networkIdle'], ['networkidle2', 'networkAlmostIdle']]); /** * @internal @@ -18012,6 +18224,10 @@ _assertClassBrand(_FrameManager_brand, this, _onFrameStoppedLoading).call(this, event.frameId); }); session.on('Runtime.executionContextCreated', async event => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } await _classPrivateFieldGet(_frameTreeHandled, this)?.valueOrThrow(); _assertClassBrand(_FrameManager_brand, this, _onExecutionContextCreated).call(this, event.context, session); }); @@ -18035,13 +18251,39 @@ _classPrivateFieldGet(_frameTreeHandled, this)?.resolve(); }), client.send('Page.setLifecycleEventsEnabled', { enabled: true - }), client.send('Runtime.enable').then(() => { - return _assertClassBrand(_FrameManager_brand, this, _createIsolatedWorld).call(this, client, UTILITY_WORLD_NAME); - }), ...(frame ? Array.from(_classPrivateFieldGet(_scriptsToEvaluateOnNewDocument, this).values()) : []).map(script => { + }), (() => { + // rebrowser-patches: skip Runtime.enable + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][FrameManager] initialize'); + return _assertClassBrand(_FrameManager_brand, this, _createIsolatedWorld).call(this, client, UTILITY_WORLD_NAME); + } + return client.send('Runtime.enable').then(() => { + return _assertClassBrand(_FrameManager_brand, this, _createIsolatedWorld).call(this, client, UTILITY_WORLD_NAME); + }); + })(), ...(frame ? Array.from(_classPrivateFieldGet(_scriptsToEvaluateOnNewDocument, this).values()) : []).map(script => { return frame?.addPreloadScript(script); }), ...(frame ? Array.from(_classPrivateFieldGet(_bindings2, this).values()) : []).map(binding => { return frame?.addExposedFunctionBinding(binding); })]); + // rebrowser-patches: manually create main world context + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + this.frames().filter(frame => { + return frame.client === client; + }).map(frame => { + const world = frame.worlds[MAIN_WORLD]; + const contextPayload = { + id: -1, + name: '', + auxData: { + frameId: frame._id + } + }; + const context = new ExecutionContext(frame.client, + // @ts-ignore + contextPayload, world); + world.setContext(context); + }); + } } catch (error) { _classPrivateFieldGet(_frameTreeHandled, this)?.resolve(); // The target might have been closed before the initialization finished. @@ -18243,6 +18485,23 @@ } this._frameTree.addFrame(frame); } + // rebrowser-patches: we cannot fully dispose contexts as they won't be recreated as we don't have Runtime events, + // instead, just mark it all empty + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches] onFrameNavigated, navigationType = ${navigationType}, id = ${framePayload.id}, url = ${framePayload.url}`); + for (const worldSymbol of [MAIN_WORLD, PUPPETEER_WORLD]) { + // @ts-ignore + if (frame?.worlds[worldSymbol].context) { + // @ts-ignore + const frameOrWorker = frame.worlds[worldSymbol].environment; + if ('clearDocumentHandle' in frameOrWorker) { + frameOrWorker.clearDocumentHandle(); + } + // @ts-ignore + frame.worlds[worldSymbol].context?.clear(worldSymbol === MAIN_WORLD ? -1 : -2); + } + } + } frame = await this._frameTree.waitForFrame(frameId); frame._navigated(framePayload); this.emit(exports.FrameManagerEvent.FrameNavigated, frame); @@ -18266,6 +18525,23 @@ frameId: frame._id, worldName: name, grantUniveralAccess: true + }).then(createIsolatedWorldResult => { + // rebrowser-patches: save created context id + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + return; + } + if (!createIsolatedWorldResult?.executionContextId) { + // probably "Target closed" error, just ignore it + return; + } + // @ts-ignore + _assertClassBrand(_FrameManager_brand, this, _onExecutionContextCreated).call(this, { + id: createIsolatedWorldResult.executionContextId, + name, + auxData: { + frameId: frame._id + } + }, frame.client); }).catch(debugError); })); _classPrivateFieldGet(_isolatedWorlds, this).add(key); @@ -20394,6 +20670,10 @@ _classPrivateFieldSet(_targetType2, this, targetType); _classPrivateFieldSet(_world4, this, new IsolatedWorld(this, new TimeoutSettings())); _classPrivateFieldGet(_client17, this).once('Runtime.executionContextCreated', async event => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } _classPrivateFieldGet(_world4, this).setContext(new ExecutionContext(client, event.context, _classPrivateFieldGet(_world4, this))); }); _classPrivateFieldGet(_world4, this).emitter.on('consoleapicalled', async event => { @@ -20411,7 +20691,21 @@ }); // This might fail if the target is closed before we receive all execution contexts. networkManager?.addClient(_classPrivateFieldGet(_client17, this)).catch(debugError); - _classPrivateFieldGet(_client17, this).send('Runtime.enable').catch(debugError); + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // @ts-ignore + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][WebWorker] initialize', targetType, targetId, client._target(), client._target()._getTargetInfo()); + // rebrowser-patches: manually create context + const contextPayload = { + id: -3, + auxData: { + frameId: targetId + } + }; + // @ts-ignore + _classPrivateFieldGet(_world4, this).setContext(new ExecutionContext(client, contextPayload, _classPrivateFieldGet(_world4, this))); + } else { + _classPrivateFieldGet(_client17, this).send('Runtime.enable').catch(debugError); + } } mainRealm() { return _classPrivateFieldGet(_world4, this); @@ -22313,6 +22607,10 @@ async newPage() { return await _classPrivateFieldGet(_defaultContext, this).newPage(); } + // rebrowser-patches: expose browser CDP session + _connection() { + return _classPrivateFieldGet(_connection4, this); + } async _createPageInContext(contextId) { const { targetId --- a/lib/esm/puppeteer/cdp/Browser.d.ts +++ b/lib/esm/puppeteer/cdp/Browser.d.ts @@ -32,6 +32,7 @@ _disposeContext(contextId?: string): Promise; wsEndpoint(): string; newPage(): Promise; + _connection(): Connection; _createPageInContext(contextId?: string): Promise; installExtension(path: string): Promise; uninstallExtension(id: string): Promise; --- a/lib/esm/puppeteer/cdp/Browser.js +++ b/lib/esm/puppeteer/cdp/Browser.js @@ -172,6 +172,10 @@ async newPage() { return await this.#defaultContext.newPage(); } + // rebrowser-patches: expose browser CDP session + _connection() { + return this.#connection; + } async _createPageInContext(contextId) { const { targetId } = await this.#connection.send('Target.createTarget', { url: 'about:blank', --- a/lib/esm/puppeteer/cdp/ExecutionContext.d.ts +++ b/lib/esm/puppeteer/cdp/ExecutionContext.d.ts @@ -22,6 +22,7 @@ bindingcalled: Protocol.Runtime.BindingCalledEvent; }> implements Disposable { #private; + _frameId: any; constructor(client: CDPSession, contextPayload: Protocol.Runtime.ExecutionContextDescription, world: IsolatedWorld); get id(): number; get puppeteerUtil(): Promise>; @@ -116,6 +117,10 @@ * {@link ElementHandle | element handle}. */ evaluateHandle = EvaluateFunc>(pageFunction: Func | string, ...args: Params): Promise>>>; + clear(newId: any): void; + __re__getMainWorld({ client, frameId, isWorker }: any): Promise; + __re__getIsolatedWorld({ client, frameId, worldName }: any): Promise; + acquireContextId(tryCount?: number): Promise; [disposeSymbol](): void; } //# sourceMappingURL=ExecutionContext.d.ts.map \ No newline at end of file --- a/lib/esm/puppeteer/cdp/ExecutionContext.js +++ b/lib/esm/puppeteer/cdp/ExecutionContext.js @@ -83,6 +83,7 @@ #client; #world; #id; + _frameId; #name; #disposables = new DisposableStack(); constructor(client, contextPayload, world) { @@ -93,16 +94,22 @@ if (contextPayload.name) { this.#name = contextPayload.name; } + // rebrowser-patches: keep frameId to use later + if (contextPayload.auxData?.frameId) { + this._frameId = contextPayload.auxData?.frameId; + } const clientEmitter = this.#disposables.use(new EventEmitter(this.#client)); clientEmitter.on('Runtime.bindingCalled', this.#onBindingCalled.bind(this)); - clientEmitter.on('Runtime.executionContextDestroyed', async (event) => { - if (event.executionContextId === this.#id) { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + clientEmitter.on('Runtime.executionContextDestroyed', async (event) => { + if (event.executionContextId === this.#id) { + this[disposeSymbol](); + } + }); + clientEmitter.on('Runtime.executionContextsCleared', async () => { this[disposeSymbol](); - } - }); - clientEmitter.on('Runtime.executionContextsCleared', async () => { - this[disposeSymbol](); - }); + }); + } clientEmitter.on('Runtime.consoleAPICalled', this.#onConsoleAPI.bind(this)); clientEmitter.on(CDPSessionEvent.Disconnected, () => { this[disposeSymbol](); @@ -325,7 +332,181 @@ async evaluateHandle(pageFunction, ...args) { return await this.#evaluate(false, pageFunction, ...args); } + // rebrowser-patches: alternative to dispose + clear(newId) { + this.#id = newId; + this.#bindings = new Map(); + this.#bindingsInstalled = false; + this.#puppeteerUtil = undefined; + } + async __re__getMainWorld({ client, frameId, isWorker = false }) { + let contextId; + // random name to make it harder to detect for any 3rd party script by watching window object and events + const randomName = [...Array(Math.floor(Math.random() * (10 + 1)) + 10)].map(() => Math.random().toString(36)[2]).join(''); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] binding name = ${randomName}`); + // add the binding + await client.send('Runtime.addBinding', { + name: randomName, + }); + // listen for 'Runtime.bindingCalled' event + const bindingCalledHandler = ({ name, payload, executionContextId }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][bindingCalledHandler]', { + name, + payload, + executionContextId + }); + if (contextId > 0) { + // already acquired the id + return; + } + if (name !== randomName) { + // ignore irrelevant bindings + return; + } + if (payload !== frameId) { + // ignore irrelevant frames + return; + } + contextId = executionContextId; + // remove this listener + client.off('Runtime.bindingCalled', bindingCalledHandler); + }; + client.on('Runtime.bindingCalled', bindingCalledHandler); + if (isWorker) { + // workers don't support `Page.addScriptToEvaluateOnNewDocument` and `Page.createIsolatedWorld`, but there are no iframes inside of them, so it's safe to just use Runtime.evaluate + await client.send('Runtime.evaluate', { + expression: `this['${randomName}']('${frameId}')`, + }); + } + else { + // we could call the binding right from `addScriptToEvaluateOnNewDocument`, but this way it will be called in all existing frames and it's hard to distinguish children from the parent + await client.send('Page.addScriptToEvaluateOnNewDocument', { + source: `document.addEventListener('${randomName}', (e) => self['${randomName}'](e.detail.frameId))`, + runImmediately: true, + }); + // create new isolated world for this frame + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + // use randomName for worldName to distinguish from normal utility world + worldName: randomName, + grantUniveralAccess: true, + }); + // emit event in the specific frame from the isolated world + await client.send('Runtime.evaluate', { + expression: `document.dispatchEvent(new CustomEvent('${randomName}', { detail: { frameId: '${frameId}' } }))`, + contextId: createIsolatedWorldResult.executionContextId, + }); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getMainWorld] result:`, { contextId }); + return contextId; + } + async __re__getIsolatedWorld({ client, frameId, worldName }) { + const createIsolatedWorldResult = await client.send('Page.createIsolatedWorld', { + frameId, + worldName, + grantUniveralAccess: true, + }); + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][getIsolatedWorld] result:`, createIsolatedWorldResult); + return createIsolatedWorldResult.executionContextId; + } + // rebrowser-patches: get context id if it's missing + async acquireContextId(tryCount = 1) { + if (this.#id > 0) { + return; + } + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] id = ${this.#id}, name = ${this.#name}, fixMode = ${fixMode}, tryCount = ${tryCount}`); + let contextId; + let tryAgain = true; + let errorMessage = 'N/A'; + if (fixMode === 'addBinding') { + try { + if (this.#id === -2) { + // isolated world + contextId = await this.__re__getIsolatedWorld({ + client: this.#client, + frameId: this._frameId, + worldName: this.#name, + }); + } + else { + // main world + contextId = await this.__re__getMainWorld({ + client: this.#client, + frameId: this._frameId, + isWorker: this.#id === -3, + }); + } + } + catch (error) { + process.env['REBROWSER_PATCHES_DEBUG'] && console.error('[rebrowser-patches][acquireContextId] error:', error); + errorMessage = error.message; + if (error instanceof Error) { + if (error.message.includes('No frame for given id found') || + error.message.includes('Target closed') || + error.message.includes('Session closed')) { + // target doesn't exist anymore, don't try again + tryAgain = false; + } + } + debugError(error); + } + } + else if (fixMode === 'alwaysIsolated') { + if (this.#id === -3) { + throw new Error('[rebrowser-patches] web workers are not supported in alwaysIsolated mode'); + } + contextId = await this.__re__getIsolatedWorld({ + client: this.#client, + frameId: this._frameId, + worldName: this.#name, + }); + } + else if (fixMode === 'enableDisable') { + const executionContextCreatedHandler = ({ context }) => { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][executionContextCreated] this.#id = ${this.#id}, name = ${this.#name}, contextId = ${contextId}, event.context.id = ${context.id}`); + if (contextId > 0) { + // already acquired the id + return; + } + if (this.#id === -1) { + // main world + if (context.auxData && context.auxData['isDefault']) { + contextId = context.id; + } + } + else if (this.#id === -2) { + // utility world + if (this.#name === context.name) { + contextId = context.id; + } + } + else if (this.#id === -3) { + // web worker + contextId = context.id; + } + }; + this.#client.on('Runtime.executionContextCreated', executionContextCreatedHandler); + await this.#client.send('Runtime.enable'); + await this.#client.send('Runtime.disable'); + this.#client.off('Runtime.executionContextCreated', executionContextCreatedHandler); + } + if (!contextId) { + if (!tryAgain || tryCount >= 3) { + throw new Error(`[rebrowser-patches] acquireContextId failed (tryAgain = ${tryAgain}, tryCount = ${tryCount}), errorMessage: ${errorMessage}`); + } + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][acquireContextId] failed, try again (tryCount = ${tryCount})`); + return this.acquireContextId(tryCount + 1); + } + this.#id = contextId; + } async #evaluate(returnByValue, pageFunction, ...args) { + // rebrowser-patches: context id is missing, acquire it and try again + if (this.#id < 0) { + await this.acquireContextId(); + // @ts-ignore + return this.#evaluate(returnByValue, pageFunction, ...args); + } const sourceUrlComment = getSourceUrlComment(getSourcePuppeteerURLIfAvailable(pageFunction)?.toString() ?? PuppeteerURL.INTERNAL_URL); if (isString(pageFunction)) { --- a/lib/esm/puppeteer/cdp/FrameManager.js +++ b/lib/esm/puppeteer/cdp/FrameManager.js @@ -151,6 +151,10 @@ this.#onFrameStoppedLoading(event.frameId); }); session.on('Runtime.executionContextCreated', async (event) => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } await this.#frameTreeHandled?.valueOrThrow(); this.#onExecutionContextCreated(event.context, session); }); @@ -175,9 +179,16 @@ this.#frameTreeHandled?.resolve(); }), client.send('Page.setLifecycleEventsEnabled', { enabled: true }), - client.send('Runtime.enable').then(() => { - return this.#createIsolatedWorld(client, UTILITY_WORLD_NAME); - }), + (() => { + // rebrowser-patches: skip Runtime.enable + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][FrameManager] initialize'); + return this.#createIsolatedWorld(client, UTILITY_WORLD_NAME); + } + return client.send('Runtime.enable').then(() => { + return this.#createIsolatedWorld(client, UTILITY_WORLD_NAME); + }); + })(), ...(frame ? Array.from(this.#scriptsToEvaluateOnNewDocument.values()) : []).map(script => { @@ -187,6 +198,26 @@ return frame?.addExposedFunctionBinding(binding); }), ]); + // rebrowser-patches: manually create main world context + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + this.frames() + .filter(frame => { + return frame.client === client; + }).map(frame => { + const world = frame.worlds[MAIN_WORLD]; + const contextPayload = { + id: -1, + name: '', + auxData: { + frameId: frame._id, + } + }; + const context = new ExecutionContext(frame.client, + // @ts-ignore + contextPayload, world); + world.setContext(context); + }); + } } catch (error) { this.#frameTreeHandled?.resolve(); @@ -355,6 +386,23 @@ } this._frameTree.addFrame(frame); } + // rebrowser-patches: we cannot fully dispose contexts as they won't be recreated as we don't have Runtime events, + // instead, just mark it all empty + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches] onFrameNavigated, navigationType = ${navigationType}, id = ${framePayload.id}, url = ${framePayload.url}`); + for (const worldSymbol of [MAIN_WORLD, PUPPETEER_WORLD]) { + // @ts-ignore + if (frame?.worlds[worldSymbol].context) { + // @ts-ignore + const frameOrWorker = frame.worlds[worldSymbol].environment; + if ('clearDocumentHandle' in frameOrWorker) { + frameOrWorker.clearDocumentHandle(); + } + // @ts-ignore + frame.worlds[worldSymbol].context?.clear(worldSymbol === MAIN_WORLD ? -1 : -2); + } + } + } frame = await this._frameTree.waitForFrame(frameId); frame._navigated(framePayload); this.emit(FrameManagerEvent.FrameNavigated, frame); @@ -382,6 +430,24 @@ worldName: name, grantUniveralAccess: true, }) + .then((createIsolatedWorldResult) => { + // rebrowser-patches: save created context id + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] === '0') { + return; + } + if (!createIsolatedWorldResult?.executionContextId) { + // probably "Target closed" error, just ignore it + return; + } + // @ts-ignore + this.#onExecutionContextCreated({ + id: createIsolatedWorldResult.executionContextId, + name, + auxData: { + frameId: frame._id, + } + }, frame.client); + }) .catch(debugError); })); this.#isolatedWorlds.add(key); --- a/lib/esm/puppeteer/cdp/IsolatedWorld.d.ts +++ b/lib/esm/puppeteer/cdp/IsolatedWorld.d.ts @@ -12,9 +12,9 @@ import type { TimeoutSettings } from '../common/TimeoutSettings.js'; import type { EvaluateFunc, HandleFor } from '../common/types.js'; import { disposeSymbol } from '../util/disposable.js'; -import type { ExecutionContext } from './ExecutionContext.js'; +import { ExecutionContext } from './ExecutionContext.js'; import type { CdpFrame } from './Frame.js'; -import type { MAIN_WORLD, PUPPETEER_WORLD } from './IsolatedWorlds.js'; +import { MAIN_WORLD, PUPPETEER_WORLD } from './IsolatedWorlds.js'; import type { CdpWebWorker } from './WebWorker.js'; /** * @internal --- a/lib/esm/puppeteer/cdp/IsolatedWorld.js +++ b/lib/esm/puppeteer/cdp/IsolatedWorld.js @@ -1,3 +1,4 @@ +//@ts-nocheck /** * @license * Copyright 2019 Google Inc. @@ -6,9 +7,11 @@ import { firstValueFrom, map, raceWith } from '../../third_party/rxjs/rxjs.js'; import { Realm } from '../api/Realm.js'; import { EventEmitter } from '../common/EventEmitter.js'; -import { fromEmitterEvent, timeout, withSourcePuppeteerURLIfNone, } from '../common/util.js'; +import { fromEmitterEvent, timeout, withSourcePuppeteerURLIfNone, UTILITY_WORLD_NAME, } from '../common/util.js'; import { disposeSymbol } from '../util/disposable.js'; import { CdpElementHandle } from './ElementHandle.js'; +import { ExecutionContext } from './ExecutionContext.js'; +import { MAIN_WORLD, PUPPETEER_WORLD } from './IsolatedWorlds.js'; import { CdpJSHandle } from './JSHandle.js'; /** * @internal @@ -67,6 +70,21 @@ * Waits for the next context to be set on the isolated world. */ async #waitForExecutionContext() { + const fixMode = process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] || 'addBinding'; + if (fixMode === 'addBinding') { + const isMainWorld = this.#frameOrWorker.worlds[MAIN_WORLD] === this; + process.env['REBROWSER_PATCHES_DEBUG'] && console.log(`[rebrowser-patches][waitForExecutionContext] frameId = ${this.#frameOrWorker._id}, isMainWorld = ${isMainWorld}`); + const contextPayload = { + id: isMainWorld ? -1 : -2, + name: isMainWorld ? '' : UTILITY_WORLD_NAME, + auxData: { + frameId: this.#frameOrWorker._id, + } + }; + const context = new ExecutionContext(this.client, contextPayload, this); + this.setContext(context); + return context; + } const error = new Error('Execution context was destroyed'); const result = await firstValueFrom(fromEmitterEvent(this.#emitter, 'context').pipe(raceWith(fromEmitterEvent(this.#emitter, 'disposed').pipe(map(() => { // The message has to match the CDP message expected by the WaitTask class. @@ -104,6 +122,8 @@ if (!context) { context = await this.#waitForExecutionContext(); } + // rebrowser-patches: make sure id is acquired + await context.acquireContextId(); const { object } = await this.client.send('DOM.resolveNode', { backendNodeId: backendNodeId, executionContextId: context.id, --- a/lib/esm/puppeteer/cdp/WebWorker.js +++ b/lib/esm/puppeteer/cdp/WebWorker.js @@ -21,6 +21,10 @@ this.#targetType = targetType; this.#world = new IsolatedWorld(this, new TimeoutSettings()); this.#client.once('Runtime.executionContextCreated', async (event) => { + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // rebrowser-patches: ignore default logic + return; + } this.#world.setContext(new ExecutionContext(client, event.context, this.#world)); }); this.#world.emitter.on('consoleapicalled', async (event) => { @@ -39,7 +43,22 @@ }); // This might fail if the target is closed before we receive all execution contexts. networkManager?.addClient(this.#client).catch(debugError); - this.#client.send('Runtime.enable').catch(debugError); + if (process.env['REBROWSER_PATCHES_RUNTIME_FIX_MODE'] !== '0') { + // @ts-ignore + process.env['REBROWSER_PATCHES_DEBUG'] && console.log('[rebrowser-patches][WebWorker] initialize', targetType, targetId, client._target(), client._target()._getTargetInfo()); + // rebrowser-patches: manually create context + const contextPayload = { + id: -3, + auxData: { + frameId: targetId, + } + }; + // @ts-ignore + this.#world.setContext(new ExecutionContext(client, contextPayload, this.#world)); + } + else { + this.#client.send('Runtime.enable').catch(debugError); + } } mainRealm() { return this.#world; --- a/lib/esm/puppeteer/common/util.js +++ b/lib/esm/puppeteer/common/util.js @@ -231,7 +231,10 @@ /** * @internal */ -export const UTILITY_WORLD_NAME = '__puppeteer_utility_world__' + packageVersion; +export const UTILITY_WORLD_NAME = +// rebrowser-patches: change utility world name +process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] !== '0' ? (process.env['REBROWSER_PATCHES_UTILITY_WORLD_NAME'] || 'util') : + '__puppeteer_utility_world__' + packageVersion; /** * @internal */ @@ -240,6 +243,10 @@ * @internal */ export function getSourceUrlComment(url) { + // rebrowser-patches: change sourceUrl to generic script name + if (process.env['REBROWSER_PATCHES_SOURCE_URL'] !== '0') { + url = process.env['REBROWSER_PATCHES_SOURCE_URL'] || 'app.js'; + } return `//# sourceURL=${url}`; } /**