import { randomUUID } from "crypto"; import { Request, Response } from "playwright"; import { Browser } from "./browserService"; import { Browser as PlaywrightBrowser } from "playwright"; import { BrowserContext as PlaywrightBrowserContext } from "playwright"; import { ElementHandle,Page} from "playwright"; import {BrowserError,BrowserState,TabInfo,URLNotAllowedError} from "./type"; import { DomService } from "../../domTypes/DomService"; import { DOMElementNode,SelectorMap } from "../../domTypes/domClass"; import path from "path"; import * as fs from "fs"; import { S3Client, PutObjectCommand, GetObjectCommand, } from "@aws-sdk/client-s3"; // Debug utility with different levels const DEBUG = { ERROR: false, // Always show errors WARN: false, // Always show warnings INFO: false, // Show general information DEBUG: false, // Show detailed debugging information TRACE: false, // Show very detailed trace information (can be verbose) error: (message: string, ...args: any[]) => { if (DEBUG.ERROR) { console.error(message, ...args); } }, warn: (message: string, ...args: any[]) => { if (DEBUG.WARN) { console.warn(message, ...args); } }, info: (message: string, ...args: any[]) => { if (DEBUG.INFO) { console.info(message, ...args); } }, debug: (message: string, ...args: any[]) => { if (DEBUG.DEBUG) { console.debug(message, ...args); } }, trace: (message: string, ...args: any[]) => { if (DEBUG.TRACE) { console.trace(message, ...args); } } }; const s3 = new S3Client(); class BrowserContextWindowSize{ width: number; height: number; constructor(width: number, height: number) { this.width = width; this.height = height; } } export class BrowserContextConfig{ cookies_file: string | null; s3Storage: boolean; minimum_wait_page_load_time: number; wait_for_network_idle_page_load_time: number; maximum_wait_page_load_time: number; wait_between_actions: number; browser_window_size: BrowserContextWindowSize; no_viewport: boolean | null; save_recording_path: string | null; save_downloads_path: string | null; trace_path: string | null; locale: string | null; user_agent: string; highlight_elements: boolean; viewport_expansion: number; allowed_domains: string[] | null; include_dynamic_attributes: boolean; disable_security: boolean; _force_keep_context_alive: boolean; constructor() { this.s3Storage=false; this.cookies_file = null; this.minimum_wait_page_load_time = 0.5; this.wait_for_network_idle_page_load_time = 1.0; this.maximum_wait_page_load_time = 5.0; this.wait_between_actions = 0.5; this.browser_window_size = new BrowserContextWindowSize(1280, 1100); this.no_viewport = null; this.save_recording_path = null; this.save_downloads_path = null; this.trace_path = null; this.locale = null; this.user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'; this.highlight_elements = true; this.viewport_expansion = 500; this.allowed_domains = null; this.include_dynamic_attributes = true; this.disable_security = true; this._force_keep_context_alive = false; } } class BrowserSession{ context: PlaywrightBrowserContext; cached_state: BrowserState | null; constructor(context: PlaywrightBrowserContext, cached_state: BrowserState | null) { this.context = context; this.cached_state = cached_state; } } class BrowserContextState{ target_id: string | null=null; constructor(target_id?: string | null) { this.target_id = target_id ?? null; } } export class BrowserContext{ context_id: string=randomUUID().toString(); config: BrowserContextConfig=new BrowserContextConfig(); current_state: BrowserState | undefined; browser: Browser=new Browser(); state?: BrowserContextState=new BrowserContextState(); session: BrowserSession | null=null; private _initialized: Promise | null = null; constructor(param?:Partial) { // Start the initialization but don't await it - will be awaited on first use this.config=param?.config?param.config:new BrowserContextConfig() this._initialized = this._initialize_session(); } // Ensure the session is initialized before any method that uses it private async _ensure_initialized(): Promise { if (this._initialized) { await this._initialized; } } async __aexit__(): Promise { await this._ensure_initialized(); await this.close(); } async close(): Promise { await this._ensure_initialized(); if(!this.session){ return } try{ if(!this.config._force_keep_context_alive && this.session.context){ await this.session.context.close(); } await this.save_cookies(); }catch(e){ DEBUG.error('Failed to close browser context', e); } this.session=null; } async save_cookies(local?: boolean):Promise{ if (this.session &&this.session.context && this.config.cookies_file){ try{ const cookies = await this.session.context.cookies(); if (local) { const dirname = path.dirname(this.config.cookies_file); if (dirname) { // Ensure the directory exists fs.mkdirSync(dirname,{recursive:true}); } // Write cookies to the file fs.writeFileSync(this.config.cookies_file, JSON.stringify(cookies, null, 2)); } else { // Upload to S3 await s3.send(new PutObjectCommand({ Bucket: process.env.AWS_S3_BUCKET_NAME, Key: path.basename(this.config.cookies_file), Body: JSON.stringify(cookies, null, 2), ContentType: 'application/json' })); } } catch (e) { DEBUG.error('Failed to save cookies', e); } } } async _initialize_session(): Promise { DEBUG.info('Initializing browser session'); try { DEBUG.debug('Getting Playwright browser instance'); const playwright_browser = await this.browser.get_playwright_browser(); DEBUG.debug('Creating browser context'); const context = await this._create_context(playwright_browser); DEBUG.debug('Getting pages from context'); const pages = context.pages(); DEBUG.debug(`Found ${pages.length} existing pages`); this.session = new BrowserSession(context, null); let active_page: Page | null = null; if (pages && pages.length > 0) { active_page = pages[0]; DEBUG.debug('Using existing page', { pageUrl: active_page.url() }); } else { DEBUG.debug('Creating new page'); active_page = await context.newPage(); DEBUG.debug('New page created'); } DEBUG.debug('Bringing active page to front'); await active_page.bringToFront(); DEBUG.debug('Waiting for page load state'); await active_page.waitForLoadState('load'); DEBUG.info('Browser context initialized successfully'); return this.session; } catch (error) { DEBUG.error('Failed to initialize session', error); throw error; } } async _add_new_page_listener(this:BrowserContext,context:PlaywrightBrowserContext){ const on_page = async (page:Page) =>{ await page.waitForLoadState() if(this.session!==null) { this.state!.target_id = null } } context.on('page',on_page) } async get_session(this:BrowserContext): Promise { await this._ensure_initialized(); if(this.session===null) { return await this._initialize_session() } return this.session } async get_current_page(this:BrowserContext): Promise { await this._ensure_initialized(); DEBUG.debug('Getting current page'); try { const session = await this.get_session(); DEBUG.debug('Session retrieved successfully'); const page = await this._get_current_page(session); DEBUG.debug('Current page retrieved successfully', { url: page.url() }); return page; } catch (error) { DEBUG.error('Failed to get current page', error); throw error; } } async _get_current_page(this:BrowserContext, session:BrowserSession): Promise { await this._ensure_initialized(); DEBUG.debug('Getting pages from browser context'); try { const pages = session.context.pages(); DEBUG.debug(`Found ${pages.length} pages in context`); if (pages.length === 0) { DEBUG.error('No pages found in context, creating a new page'); const newPage = await session.context.newPage(); return newPage; } const currentPage = pages[pages.length - 1]; DEBUG.debug('Selected current page', { url: currentPage.url(), index: pages.length - 1, totalPages: pages.length }); return currentPage; } catch (error) { DEBUG.error('Error getting current page from session', error); throw error; } } async _create_context(this:BrowserContext,browser:PlaywrightBrowser): Promise { if(this.browser.config.browser_instance_path && (await browser.contexts()).length > 0){ return (await browser.contexts())[0] } else{ const context = await browser.newContext( { viewport:this.config.browser_window_size, javaScriptEnabled:true, bypassCSP:this.config.disable_security, ignoreHTTPSErrors:this.config.disable_security, recordVideo:{dir:this.config.save_recording_path||"null", size:this.config.browser_window_size }, locale:this.config.locale||undefined, userAgent:this.config.user_agent } ) if(this.config.trace_path){ (await context).tracing.start({screenshots:true,snapshots:true,sources:true}) } //Add Cookies if(this.config.cookies_file){ //Load cookies from s3 if(this.config.s3Storage){ const cookiesFromS3=await s3.send( new GetObjectCommand({ Bucket: process.env.AWS_S3_BUCKET_NAME, Key: path.basename(this.config.cookies_file) }) ) const cookies=JSON.parse((await cookiesFromS3)?.Body?.toString() || '') await (await context).addCookies(cookies) } //Or locally else{ const cookies=fs.readFileSync(this.config.cookies_file,'utf8') await (await context).addCookies(JSON.parse(cookies)) } } //Expose Anti-Detection Scripts (await context).addInitScript({ content:` // Webdriver property Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); // Languages Object.defineProperty(navigator, 'languages', { get: () => ['en-US'] }); // Plugins Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); // Chrome runtime window.chrome = { runtime: {} }; // Permissions const originalQuery = window.navigator.permissions.query; window.navigator.permissions.query = (parameters) => ( parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters) ); (function () { const originalAttachShadow = Element.prototype.attachShadow; Element.prototype.attachShadow = function attachShadow(options) { return originalAttachShadow.call(this, { ...options, mode: "open" }); }; })(); ` }) return context } } async _wait_for_stable_network(this:BrowserContext): Promise { const relevant_resource_types = new Set(['document', 'script', 'stylesheet', 'xhr', 'fetch', 'other']); const relevant_content_types = new Set(['text/html', 'application/javascript', 'text/css']); const ignored_url_patterns = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf']); const page = await this.get_current_page(); const pending_requests = new Set(); let last_activity = Date.now(); // Function to handle request events const on_request = (request: Request): void => { const url = request.url().toLowerCase(); if (!relevant_resource_types.has(request.resourceType())) { return; } if (ignored_url_patterns.has(url)) { return; } pending_requests.add(request); last_activity = Date.now(); }; // Function to handle response events const on_response = (response: Response): void => { const request = response.request(); if (!pending_requests.has(request)) { return; } const content_type = response.headers()['content-type']?.toLowerCase() || ''; if (!relevant_content_types.has(content_type)) { pending_requests.delete(request); return; } pending_requests.delete(request); last_activity = Date.now(); }; // Attach event listeners to the page page.on('request', on_request); page.on('response', on_response); try { // Wait for the network to stabilize let timeout = false; while (!timeout) { await new Promise(resolve => setTimeout(resolve, 100)); // Sleep for 100ms const now = Date.now(); if (pending_requests.size === 0 && (now - last_activity) >= this.config.wait_for_network_idle_page_load_time * 1000) { break; } if (now - last_activity > this.config.maximum_wait_page_load_time * 1000) { timeout = true; break; } } } finally { // Clean up event listeners page.removeListener('request', on_request); page.removeListener('response', on_response); } } async _wait_for_page_and_frames_load(this:BrowserContext, timeout_overwrite: number | null = null): Promise { await this._ensure_initialized(); const start_time=Date.now() try{ await this._wait_for_stable_network() const page = await this.get_current_page() await this._check_and_handle_navigation(page) } catch(e){ DEBUG.error('Error waiting for page and frames load', e); } const elapsed = (Date.now() - start_time) / 1000 const remaining = Math.max((timeout_overwrite || this.config.minimum_wait_page_load_time) - elapsed, 0) if (remaining > 0) { await new Promise(resolve => setTimeout(resolve, remaining * 1000)) } } async _is_url_allowed(this:BrowserContext, url: string): Promise { await this._ensure_initialized(); if (!this.config.allowed_domains) { return true } try { const parsed_url = new URL(url) const domain = parsed_url.hostname.toLowerCase() return this.config.allowed_domains.some(allowed_domain => { return domain === allowed_domain.toLowerCase() || domain.endsWith(`.${allowed_domain.toLowerCase()}`) }) } catch (e) { DEBUG.error('Error checking URL allowed', e); return false } } async _check_and_handle_navigation(this:BrowserContext, page:Page): Promise { await this._ensure_initialized(); DEBUG.debug('Starting navigation check', { url: page.url() }); try { if (!page) { DEBUG.error('No page provided for navigation check'); throw new Error('No page provided for navigation check'); } DEBUG.debug('Waiting for navigation to complete'); await page.waitForLoadState('domcontentloaded'); await page.waitForLoadState('load', { timeout: 10000 }).catch(e => { DEBUG.warn('Load state timeout', e); }); DEBUG.debug('Checking current URL', { url: page.url() }); const url = page.url(); // Check if URL is allowed if (!this._is_url_allowed(url)) { DEBUG.warn(`Navigation to non-allowed URL detected: ${url}`); try { await this.go_back(); } catch (e) { DEBUG.error(`Failed to go back after detecting non-allowed URL: ${e}`); } throw new URLNotAllowedError(`Navigation to non-allowed URL: ${url}`); } // Handle various navigation scenarios if (url.startsWith('chrome-error://')) { DEBUG.warn('Chrome error page detected', { url }); // Handle chrome error } DEBUG.debug('Navigation check completed successfully'); } catch (error) { if (!(error instanceof URLNotAllowedError)) { DEBUG.error('Error during navigation check', error); } throw error; } } async handle_page_load(this:BrowserContext): Promise{ await this._ensure_initialized(); DEBUG.info('Starting page load handling'); const start_time=Date.now() try{ DEBUG.debug('Waiting for page to load and become stable'); await this._wait_for_page_and_frames_load(); DEBUG.debug('Page is stable, updating state'); const page = await this.get_current_page(); DEBUG.debug('Checking for navigation events'); await this._check_and_handle_navigation(page); await this._wait_for_stable_network(); DEBUG.info('Navigation check completed successfully'); } catch(e){ DEBUG.error('Page load failed', e); // Don't return anything for void function } const end_time = Date.now(); DEBUG.debug(`Page load handling completed in ${end_time - start_time}ms`); } async navigate_to(this:BrowserContext, url: string): Promise { await this._ensure_initialized(); if (!this._is_url_allowed(url)) { throw new Error(`Navigation to non-allowed URL: ${url}`) } const page = await this.get_current_page() await page.goto(url) await page.waitForLoadState() } async refresh_page(this:BrowserContext): Promise { await this._ensure_initialized(); const page = await this.get_current_page() await page.reload() await page.waitForLoadState() } async go_back(this:BrowserContext): Promise { await this._ensure_initialized(); const page = await this.get_current_page() try { await page.goBack({ timeout: 10, waitUntil: 'domcontentloaded' }) } catch (e) { DEBUG.error('Failed to go back', e); } } async go_forward(this:BrowserContext): Promise { await this._ensure_initialized(); const page = await this.get_current_page() try { await page.goForward({ timeout: 10, waitUntil: 'domcontentloaded' }) } catch (e) { DEBUG.error('Failed to go forward', e); } } async close_current_tab(this:BrowserContext): Promise { await this._ensure_initialized(); const session=this.session const page=await this._get_current_page(session!) await page.close() if(session?.context?.pages){ await this.switch_to_tab(0) } } async get_page_html(this:BrowserContext): Promise { await this._ensure_initialized(); const page = await this.get_current_page() return await page.content() } async execute_javascript(this:BrowserContext, script: string): Promise { await this._ensure_initialized(); const page = await this.get_current_page() return await page.evaluate(script) } async get_page_structure(this:BrowserContext): Promise { await this._ensure_initialized(); const debug_script=` (() => { function getPageStructure(element = document, depth = 0, maxDepth = 10) { if (depth >= maxDepth) return ''; const indent = ' '.repeat(depth); let structure = ''; // Skip certain elements that clutter the output const skipTags = new Set(['script', 'style', 'link', 'meta', 'noscript']); // Add current element info if it's not the document if (element !== document) { const tagName = element.tagName.toLowerCase(); // Skip uninteresting elements if (skipTags.has(tagName)) return ''; const id = element.id ? \`#\${element.id}\` : ''; const classes = element.className && typeof element.className === 'string' ? \`.\${element.className.split(' ').filter(c => c).join('.')}\` : ''; // Get additional useful attributes const attrs = []; if (element.getAttribute('role')) attrs.push(\`role="\${element.getAttribute('role')}"\`); if (element.getAttribute('aria-label')) attrs.push(\`aria-label="\${element.getAttribute('aria-label')}"\`); if (element.getAttribute('type')) attrs.push(\`type="\${element.getAttribute('type')}"\`); if (element.getAttribute('name')) attrs.push(\`name="\${element.getAttribute('name')}"\`); if (element.getAttribute('src')) { const src = element.getAttribute('src'); attrs.push(\`src="\${src.substring(0, 50)}\${src.length > 50 ? '...' : ''}"\`); } // Add element info structure += \`\${indent}\${tagName}\${id}\${classes}\${attrs.length ? ' [' + attrs.join(', ') + ']' : ''}\\n\`; // Handle iframes specially if (tagName === 'iframe') { try { const iframeDoc = element.contentDocument || element.contentWindow?.document; if (iframeDoc) { structure += \`\${indent} [IFRAME CONTENT]:\\n\`; structure += getPageStructure(iframeDoc, depth + 2, maxDepth); } else { structure += \`\${indent} [IFRAME: No access - likely cross-origin]\\n\`; } } catch (e) { } } } // Get all child elements const children = element.children || element.childNodes; for (const child of children) { if (child.nodeType === 1) { // Element nodes only structure += getPageStructure(child, depth + 1, maxDepth); } } return structure; } return getPageStructure(); })() ` return (await this.get_current_page()).evaluate(debug_script) } async get_state(this:BrowserContext):Promise{ await this._ensure_initialized(); await this._wait_for_page_and_frames_load() const session=await this.get_session() session.cached_state=await this._update_state() if (this.config.cookies_file){ await this.save_cookies(true) } else if(this.config.s3Storage){ await this.save_cookies(false) } return session.cached_state! } async _update_state(this:BrowserContext, focus_element: number = -1):Promise{ await this._ensure_initialized(); const session=await this.get_session() let page:Page|undefined=undefined try{ page=await this.get_current_page() await page.evaluate('1+1') } catch(e){ const pages=await session.context.pages() if(pages){ this.state!.target_id=null page=await this.get_current_page() } else{ throw new BrowserError(`Browser closed: no valid pages available ${e}`) } } try{ await this.remove_highlights() const dom_service = new DomService(page) const content = await dom_service.getClickableElements( this.config.highlight_elements, focus_element, this.config.viewport_expansion, ) const screenshot_b64 = await this.take_screenshot() const [pixels_above, pixels_below] = await this.get_scroll_info(page!) this.current_state = new BrowserState( content.elementTree, content.selectorMap, await page!.url(), await page!.title(), await this.get_tabs_info(), screenshot_b64, pixels_above, pixels_below, ) return this.current_state } catch(e){ // Return last known good state if available if (this.current_state) { return this.current_state } throw e } } async take_screenshot(this:BrowserContext, full_page: boolean = false): Promise { await this._ensure_initialized(); const page = await this.get_current_page() await page.bringToFront() await page.waitForLoadState() const screenshot:Buffer = await page.screenshot({ fullPage:full_page, animations:'disabled' }) const screenshot_b64 = screenshot.toString('base64') return screenshot_b64 } async remove_highlights(this:BrowserContext): Promise { await this._ensure_initialized(); try { const page = await this.get_current_page() await page.evaluate( ` try { // Remove the highlight container and all its contents const container = document.getElementById('playwright-highlight-container'); if (container) { container.remove(); } // Remove highlight attributes from elements const highlightedElements = document.querySelectorAll('[browser-user-highlight-id^="playwright-highlight-"]'); highlightedElements.forEach(el => { el.removeAttribute('browser-user-highlight-id'); }); } catch (e) { } ` ) } catch (e) { DEBUG.error('Error removing highlights', e); // Don't raise the error since this is not critical functionality } } async _convert_simple_xpath_to_css_selector(xpath: string): Promise { /** * Converts simple XPath expressions to CSS selectors. * * @param xpath - The XPath expression to convert. * @returns A CSS selector string. */ if (!xpath) { return ''; } // Remove leading slash if present xpath = xpath.replace(/^\//, ''); // Split into parts const parts = xpath.split('/'); const css_parts: string[] = []; for (const part of parts) { if (!part) { continue; } // Handle custom elements with colons by escaping them if (part.includes(':') && !part.includes('[')) { const base_part = part.replace(/:/g, '\\:'); css_parts.push(base_part); continue; } // Handle index notation [n] if (part.includes('[')) { let base_part = part.slice(0, part.indexOf('[')); // Handle custom elements with colons in the base part if (base_part.includes(':')) { base_part = base_part.replace(/:/g, '\\:'); } const index_part = part.slice(part.indexOf('[')); // Handle multiple indices const indices = index_part.split(']').slice(0, -1).map(i => i.replace(/\[|\]/g, '')); for (const idx of indices) { try { // Handle numeric indices if (!isNaN(Number(idx))) { const index = parseInt(idx, 10) - 1; base_part += `:nth-of-type(${index + 1})`; // Handle last() function } else if (idx === 'last()') { base_part += ':last-of-type'; // Handle position() functions } else if (idx.includes('position()')) { if (idx.includes('>1')) { base_part += ':nth-of-type(n+2)'; } } } catch (e) { DEBUG.error('Error enhancing CSS selector', e); continue; } } css_parts.push(base_part); } else { css_parts.push(part); } } return css_parts.join(' > '); } async _enhanced_css_selector_for_element(element: DOMElementNode, include_dynamic_attributes: boolean = true): Promise { /** * Creates a CSS selector for a DOM element, handling various edge cases and special characters. * * @param element - The DOM element to create a selector for. * @param include_dynamic_attributes - Whether to include dynamic attributes in the selector. * @returns A valid CSS selector string. */ try { // Get base selector from XPath let css_selector = await this._convert_simple_xpath_to_css_selector(element.xpath); // Handle class attributes if ('class' in element.attributes && element.attributes['class'] && include_dynamic_attributes) { // Define a regex pattern for valid class names in CSS const valid_class_name_pattern = /^[a-zA-Z_][a-zA-Z0-9_-]*$/; // Iterate through the class attribute values const classes = element.attributes['class'].split(' '); for (const class_name of classes) { // Skip empty class names if (!class_name.trim()) { continue; } // Check if the class name is valid if (valid_class_name_pattern.test(class_name)) { // Append the valid class name to the CSS selector css_selector += `.${class_name}`; } } } // Expanded set of safe attributes that are stable and useful for selection const SAFE_ATTRIBUTES = new Set([ 'id', 'name', 'type', 'placeholder', 'aria-label', 'aria-labelledby', 'aria-describedby', 'role', 'for', 'autocomplete', 'required', 'readonly', 'alt', 'title', 'src', 'href', 'target', ]); if (include_dynamic_attributes) { const dynamic_attributes = ['data-id', 'data-qa', 'data-cy', 'data-testid']; dynamic_attributes.forEach(attr => SAFE_ATTRIBUTES.add(attr)); } // Handle other attributes for (const [attribute, value] of Object.entries(element.attributes)) { if (attribute === 'class') { continue; } // Skip invalid attribute names if (!attribute.trim()) { continue; } if (!SAFE_ATTRIBUTES.has(attribute)) { continue; } // Escape special characters in attribute names const safe_attribute = attribute.replace(/:/g, '\\:'); // Handle different value cases if (value === '') { css_selector += `[${safe_attribute}]`; } else if (/["'<>`\n\r\t]/.test(value)) { // Use contains for values with special characters const collapsed_value = value.replace(/\s+/g, ' ').trim(); const safe_value = collapsed_value.replace(/"/g, '\\"'); css_selector += `[${safe_attribute}*="${safe_value}"]`; } else { css_selector += `[${safe_attribute}="${value}"]`; } } return css_selector; } catch (e) { DEBUG.error('Error enhancing CSS selector, Falling back to basic one', e); // Fallback to a more basic selector if something goes wrong const tag_name = element.tagName || '*'; return `${tag_name}[highlight_index='${element.highlightIndex}']`; } } async get_locate_element(this:BrowserContext, element: DOMElementNode): Promise { await this._ensure_initialized(); if (!element) { DEBUG.error('Element is null or undefined in get_locate_element'); return null; } let current_frame:any=await(this.get_current_page()) let current=element const parent:Array=[] while(current.parent!==null){ parent.push(current.parent) current=current.parent } parent.reverse() const iframes = parent.filter(item => item.tagName === 'iframe'); for(const iframe of iframes){ const css_selector=await this._convert_simple_xpath_to_css_selector(iframe.xpath) current_frame=current_frame.frameLocator(css_selector) } // Generate CSS selector from XPath const css_selector=await this._convert_simple_xpath_to_css_selector(element.xpath) DEBUG.info(`Attempting to locate element with xpath: ${element.xpath}`); DEBUG.info(`Converted CSS selector: ${css_selector}`); // Try multiple strategies to locate the element try { // STRATEGY 1: Try locating with the converted CSS selector if ('frameLocator' in current_frame && typeof current_frame.frameLocator === 'function') { // Try with locator first DEBUG.info(`Strategy 1: Trying to locate using frameLocator.locator(${css_selector})`); try { const element_handle = await current_frame.locator(css_selector).elementHandle({ timeout: 2000 }); if (element_handle) { DEBUG.info(`Successfully found element with locator`); return element_handle; } } catch (locatorErr:any) { DEBUG.warn(`Failed to locate using CSS selector: ${locatorErr.message}`); } // STRATEGY 2: Try using enhanced CSS selector with attributes DEBUG.info('Strategy 2: Trying enhanced CSS selector with attributes'); try { const enhanced_selector = await this._enhanced_css_selector_for_element(element, true); if (enhanced_selector !== css_selector) { DEBUG.info(`Enhanced CSS selector: ${enhanced_selector}`); const element_handle = await current_frame.locator(enhanced_selector).elementHandle({ timeout: 2000 }); if (element_handle) { DEBUG.info(`Successfully found element with enhanced CSS selector`); return element_handle; } } } catch (enhancedSelectorErr:any) { DEBUG.warn(`Failed to locate using enhanced CSS selector: ${enhancedSelectorErr.message}`); } // STRATEGY 3: Try using text content if available if (element.attributes && 'textContent' in element.attributes && element.attributes['textContent'] && element.attributes['textContent'].trim()) { DEBUG.info(`Strategy 3: Trying to locate using text content: "${element.attributes['textContent'].trim()}"`); try { const trimmedText = element.attributes['textContent'].trim().substring(0, 50); // Use first 50 chars to avoid overly long text const element_handle = await current_frame.locator(`text=${trimmedText}`).elementHandle({ timeout: 2000 }); if (element_handle) { DEBUG.info(`Successfully found element with text content`); return element_handle; } } catch (textErr:any) { DEBUG.warn(`Failed to locate using text content: ${textErr.message}`); } } // STRATEGY 4: Try using nth child as a fallback if highlight index is present if (element.highlightIndex !== null && element.highlightIndex !== undefined) { DEBUG.info(`Strategy 4: Trying to locate using highlightIndex: ${element.highlightIndex}`); try { // Try by tag name and index const tagSelector = `${element.tagName}:nth-child(${element.highlightIndex})`; const element_handle = await current_frame.locator(tagSelector).first().elementHandle({ timeout: 2000 }); if (element_handle) { DEBUG.info(`Successfully found element with tag selector: ${tagSelector}`); return element_handle; } } catch (indexErr:any) { DEBUG.warn(`Failed to locate using highlightIndex: ${indexErr.message}`); } // Try locating by attribute try { const attributeSelector = `[highlight_index='${element.highlightIndex}']`; const element_handle = await current_frame.locator(attributeSelector).elementHandle({ timeout: 2000 }); if (element_handle) { DEBUG.info(`Successfully found element with highlight_index attribute`); return element_handle; } } catch (attrErr:any) { DEBUG.warn(`Failed to locate using highlight_index attribute: ${attrErr.message}`); } } DEBUG.error(`All strategies failed to locate element with xpath: ${element.xpath}`); return null; } else { // STRATEGY 1: Try querySelector with CSS selector DEBUG.info(`Strategy 1: Trying to locate using page.querySelector(${css_selector})`); try { const element_handle = await current_frame.querySelector(css_selector); if (element_handle) { await element_handle.scrollIntoViewIfNeeded(); DEBUG.info(`Successfully found element with querySelector`); return element_handle; } } catch (querySelectorErr:any) { DEBUG.warn(`Failed to locate using querySelector: ${querySelectorErr.message}`); } // STRATEGY 2: Try using enhanced CSS selector with attributes DEBUG.info('Strategy 2: Trying enhanced CSS selector with attributes'); try { const enhanced_selector = await this._enhanced_css_selector_for_element(element, true); if (enhanced_selector !== css_selector) { DEBUG.info(`Enhanced CSS selector: ${enhanced_selector}`); const element_handle = await current_frame.querySelector(enhanced_selector); if (element_handle) { await element_handle.scrollIntoViewIfNeeded(); DEBUG.info(`Successfully found element with enhanced CSS selector`); return element_handle; } } } catch (enhancedSelectorErr:any) { DEBUG.warn(`Failed to locate using enhanced CSS selector: ${enhancedSelectorErr.message}`); } // STRATEGY 3: Try evaluating xpath directly DEBUG.info(`Strategy 3: Trying to evaluate XPath directly: ${element.xpath}`); try { const elements = await current_frame.$$eval(`xpath=${element.xpath}`, (nodes: any[]) => nodes.length > 0); if (elements) { const element_handle = await current_frame.$(`xpath=${element.xpath}`); if (element_handle) { await element_handle.scrollIntoViewIfNeeded(); DEBUG.info(`Successfully found element with direct XPath evaluation`); return element_handle; } } } catch (xpathErr:any) { DEBUG.warn(`Failed to locate using direct XPath evaluation: ${xpathErr.message}`); } // STRATEGY 4: Try using nth child as a fallback if highlight index is present if (element.highlightIndex !== null && element.highlightIndex !== undefined) { DEBUG.info(`Strategy 4: Trying to locate using highlightIndex: ${element.highlightIndex}`); try { // Try by tag name and index const tagSelector = `${element.tagName}:nth-child(${element.highlightIndex})`; const element_handle = await current_frame.querySelector(tagSelector); if (element_handle) { await element_handle.scrollIntoViewIfNeeded(); DEBUG.info(`Successfully found element with tag selector: ${tagSelector}`); return element_handle; } } catch (indexErr:any) { DEBUG.warn(`Failed to locate using highlightIndex: ${indexErr.message}`); } // Try locating by attribute try { const attributeSelector = `[highlight_index='${element.highlightIndex}']`; const element_handle = await current_frame.querySelector(attributeSelector); if (element_handle) { await element_handle.scrollIntoViewIfNeeded(); DEBUG.info(`Successfully found element with highlight_index attribute`); return element_handle; } } catch (attrErr:any) { DEBUG.warn(`Failed to locate using highlight_index attribute: ${attrErr.message}`); } } DEBUG.error(`All strategies failed to locate element with xpath: ${element.xpath}`); return null; } } catch (e:any) { DEBUG.error(`Error in get_locate_element: ${e.message}`); DEBUG.error(`Element details - tagName: ${element.tagName}, xpath: ${element.xpath}, highlightIndex: ${element.highlightIndex}`); return null; } } async get_locate_element_by_xpath(this:BrowserContext, xpath: string): Promise { await this._ensure_initialized(); DEBUG.debug(`Locating element by XPath: ${xpath}`); const currentFrame = await this.get_current_page(); try { // Use XPath to locate the element const elementHandle = await currentFrame.locator(`xpath=${xpath}`).elementHandle(); if (elementHandle) { await elementHandle.scrollIntoViewIfNeeded(); return elementHandle; } return null; } catch (e) { DEBUG.error(`Failed to locate element by XPath ${xpath}: ${e}`); return null; } } async get_locate_element_by_css_selector(this:BrowserContext, css_selector: string): Promise { await this._ensure_initialized(); DEBUG.debug(`Locating element by CSS selector: ${css_selector}`); const currentFrame = await this.get_current_page(); try { // Use CSS selector to locate the element const elementHandle = await currentFrame.locator(css_selector).elementHandle(); if (elementHandle) { await elementHandle.scrollIntoViewIfNeeded(); return elementHandle; } return null; } catch (e) { DEBUG.error(`Failed to locate element by CSS selector ${css_selector}: ${e}`); return null; } } async get_locate_element_by_text(this:BrowserContext, text: string, nth: number = 0): Promise { await this._ensure_initialized(); DEBUG.debug(`Locating element by text: "${text}" (nth: ${nth})`); const currentFrame = await this.get_current_page(); try { const elements = await currentFrame.locator(`text=${text}`).elementHandles(); // Filter to get only visible elements const visibleElements: ElementHandle[] = []; for (const el of elements) { if (await el.isVisible()) { visibleElements.push(el); } } if (visibleElements.length === 0) { DEBUG.error(`No visible element with text '${text}' found.`); return null; } if (nth !== null && nth !== undefined) { if (0 <= nth && nth < visibleElements.length) { const elementHandle = visibleElements[nth]; await elementHandle.scrollIntoViewIfNeeded(); return elementHandle; } else { DEBUG.error(`Visible element with text '${text}' not found at index ${nth}.`); return null; } } else { const elementHandle = visibleElements[0]; await elementHandle.scrollIntoViewIfNeeded(); return elementHandle; } } catch (e) { DEBUG.error(`Failed to locate element by text '${text}': ${e}`); return null; } } async _input_text_element_node(this:BrowserContext, element_node: DOMElementNode, text: string): Promise { await this._ensure_initialized(); try{ await this._update_state(element_node.highlightIndex||undefined) const element_handle=await this.get_locate_element(element_node) if(element_handle===null){ throw new BrowserError(`Element: ${element_node} not found`) } try{ await element_handle.waitForElementState('stable',{timeout:1000}) await element_handle.scrollIntoViewIfNeeded({timeout:1000}) } catch(scrollError:any){ DEBUG.warn(`Unable to stabilize or scroll element: ${scrollError.message}`); } const tag_handle=await element_handle.getProperty('tagName') const tag_name=(await tag_handle.jsonValue()).toLowerCase() const is_contenteditable=await element_handle.getProperty('isContentEditable') const readonly_handle=await element_handle.getProperty('readOnly') const disabled_handle=await element_handle.getProperty('disabled') const readonly=(await readonly_handle.jsonValue()) as boolean const disabled=(await disabled_handle.jsonValue()) as boolean // Fix: Properly await and separate the content editable check const isContentEditable = await is_contenteditable.jsonValue(); DEBUG.info(`Element input details: tag=${tag_name}, contentEditable=${isContentEditable}, readonly=${readonly}, disabled=${disabled}`); if((isContentEditable || tag_name==='input') && !(readonly||disabled)){ await element_handle.evaluate('el=>el.textContent=""') await element_handle.type(text,{delay:5}) } else{ await element_handle.fill(text) } } catch(e:any){ DEBUG.error(`Input text error for element index ${element_node.highlightIndex}:`, e); throw new BrowserError(`Failed to input text into element index ${element_node.highlightIndex}: ${e.message}`) } } async _click_element_node(this:BrowserContext, element_node: DOMElementNode){ await this._ensure_initialized(); const page = await this.get_current_page(); try { // First update state and highlight the element await this._update_state(element_node.highlightIndex || undefined); // Locate the element with more detailed diagnostics DEBUG.info(`Attempting to locate element for clicking with xpath: ${element_node.xpath}`); const element_handle = await this.get_locate_element(element_node); if (element_handle === null) { DEBUG.error(`Element not found for clicking: ${element_node.xpath}`); throw new BrowserError(`Element not found for clicking. Element details - tagName: ${element_node.tagName}, xpath: ${element_node.xpath}, highlightIndex: ${element_node.highlightIndex}`); } // Check element state before clicking const isVisible = await element_handle.isVisible(); if (!isVisible) { DEBUG.warn(`Element is not visible, attempting to scroll it into view`); } // Check if element is enabled let isEnabled = true; try { isEnabled = await element_handle.evaluate(el => { if ('disabled' in el) { return !(el as any).disabled; } return true; }); } catch (enabledErr:any) { DEBUG.warn(`Unable to check if element is enabled: ${enabledErr.message}`); } if (!isEnabled) { DEBUG.warn(`Element appears to be disabled but still attempting to click`); } // Try to ensure element is stable and scrolled into view try { await element_handle.waitForElementState('stable', { timeout: 2000 }); await element_handle.scrollIntoViewIfNeeded({ timeout: 2000 }); } catch (scrollError:any) { DEBUG.warn(`Unable to stabilize or scroll element: ${scrollError.message}. Will still attempt to click.`); } // Define the perform_click function with the click logic const perform_click = async (click_func: () => Promise) => { if (this.config.save_downloads_path) { try { // Handle downloads if configured DEBUG.info('Checking for potential download on click'); const download_info = await page.waitForEvent('download', { timeout: 5000 }); await click_func(); // Save the downloaded file const suggested_filename = download_info.suggestedFilename(); const unique_filename = await this._get_unique_filename(this.config.save_downloads_path, suggested_filename); const download_path = path.join(this.config.save_downloads_path, unique_filename); await download_info.saveAs(download_path); DEBUG.info(`Downloaded file saved to: ${download_path}`); return download_path; } catch (downloadError:any) { // If no download happens, treat as a normal click DEBUG.info('No download triggered, proceeding with normal click, error: ' + downloadError.message); await click_func(); await page.waitForLoadState(); await this._check_and_handle_navigation(page); } } else { // Standard click logic await click_func(); await page.waitForLoadState(); await this._check_and_handle_navigation(page); } }; // First attempt: standard click with retry try { DEBUG.info('Attempting standard element click'); return await perform_click(async () => { // Try up to 3 times with increasing timeout let lastError; for (let attempt = 1; attempt <= 3; attempt++) { try { DEBUG.info(`Click attempt ${attempt}/3`); await element_handle.click({ timeout: attempt * 1000 }); DEBUG.info('Click succeeded'); return; } catch (e:any) { lastError = e; DEBUG.warn(`Click attempt ${attempt} failed: ${e.message}`); // Wait briefly before retrying await new Promise(resolve => setTimeout(resolve, 500)); } } // If we get here, all attempts failed throw lastError; }); } catch (e:any) { if (e instanceof URLNotAllowedError) { throw e; } DEBUG.warn(`Standard click failed: ${e.message}, trying JavaScript click method`); // Second attempt: JavaScript click() method try { return await perform_click(async () => { DEBUG.info('Attempting JavaScript click method'); await page.evaluate('(el) => el.click()', element_handle); DEBUG.info('JavaScript click succeeded'); }); } catch (e2:any) { // Third attempt: Force click with position DEBUG.warn(`JavaScript click failed: ${e2.message}, trying position-based click`); try { return await perform_click(async () => { const box = await element_handle.boundingBox(); if (box) { DEBUG.info(`Attempting position-based click at x:${box.x + box.width/2}, y:${box.y + box.height/2}`); await page.mouse.click(box.x + box.width/2, box.y + box.height/2); DEBUG.info('Position-based click succeeded'); } else { throw new Error('Could not determine element position for click'); } }); } catch (e3:any) { DEBUG.error(`All click methods failed: ${e3.message}`); throw new BrowserError(`Failed to click element after multiple attempts: ${e3.message}`); } } } } catch (e:any) { DEBUG.error(`Click error: ${e.message}`); if (e instanceof BrowserError) { throw e; } throw new BrowserError(`Failed to click element: ${e.message}`); } } async get_tabs_info(this:BrowserContext): Promise { await this._ensure_initialized(); const session = await this.get_session() const tabs_info = await Promise.all(session.context.pages().map(async (page, page_id) => new TabInfo(page_id, page.url(), await page.title()))) return tabs_info } async switch_to_tab(this:BrowserContext, page_id: number): Promise { await this._ensure_initialized(); const session=await this.get_session() const pages=session.context.pages() if(page_id>=pages.length){ throw new BrowserError(`No tab found with page_id: ${page_id}`) } const page=pages[page_id] await page.bringToFront() await page.waitForLoadState() await this._check_and_handle_navigation(page) } async create_new_tab(this:BrowserContext, url: string | null = null): Promise { await this._ensure_initialized(); const session = await this.get_session() const new_page = await session.context.newPage() await new_page.waitForLoadState() if (url) { await new_page.goto(url) await this._wait_for_page_and_frames_load() } } async get_selector_map(this:BrowserContext): Promise { await this._ensure_initialized(); const session = await this.get_session() if (session.cached_state === null) { return {} } return session.cached_state.selectorMap } async get_element_by_index(this:BrowserContext, index: number): Promise { await this._ensure_initialized(); const selector_map = await this.get_selector_map() const element_handle = await this.get_locate_element(selector_map[index]) return element_handle } async get_dom_element_by_index(this:BrowserContext, index: number): Promise { await this._ensure_initialized(); const selector_map = await this.get_selector_map() return selector_map[index] } async is_file_uploader(this:BrowserContext, element_node: DOMElementNode, max_depth: number = 3, current_depth: number = 0): Promise { await this._ensure_initialized(); if (current_depth > max_depth) { return false } let is_uploader = false if (!(element_node instanceof DOMElementNode)) { return false } // Check for file input attributes if (element_node.tagName === 'input') { is_uploader = element_node.attributes['type'] === 'file' || (Object.prototype.hasOwnProperty.call(element_node.attributes, 'accept') && element_node.attributes['accept'] !== null) } // Recursively check children if (element_node.children && current_depth < max_depth) { for (const child of element_node.children) { if (await this.is_file_uploader(child as DOMElementNode, max_depth, current_depth + 1)) { return true } } } return is_uploader } async get_scroll_info(this:BrowserContext, page: Page): Promise<[number, number]> { await this._ensure_initialized(); const scroll_y:number = await page.evaluate('window.scrollY') const viewport_height:number = await page.evaluate('window.innerHeight') const total_height:number = await page.evaluate('document.documentElement.scrollHeight') const pixels_above:number = scroll_y const pixels_below:number = total_height - (scroll_y + viewport_height) return [pixels_above, pixels_below] } async reset_context(this:BrowserContext): Promise { await this._ensure_initialized(); const session = await this.get_session() const pages = session.context.pages() for (const page of pages) { await page.close() } session.cached_state = null this.state!.target_id = "" } async _get_unique_filename(this:BrowserContext, directory: string, filename: string): Promise { await this._ensure_initialized(); const base = path.basename(filename, path.extname(filename)); const ext = path.extname(filename); let counter = 1; let new_filename = filename; while (fs.existsSync(path.join(directory, new_filename))) { new_filename = `${base} (${counter})${ext}`; counter += 1; } return new_filename; } }