"use strict"; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; var __getOwnPropNames = Object.getOwnPropertyNames; var __hasOwnProp = Object.prototype.hasOwnProperty; var __export = (target, all) => { for (var name in all) __defProp(target, name, { get: all[name], enumerable: true }); }; var __copyProps = (to, from, except, desc) => { if (from && typeof from === "object" || typeof from === "function") { for (let key of __getOwnPropNames(from)) if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); } return to; }; var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); // src/index.ts var src_exports = {}; __export(src_exports, { BrowserUtils: () => BrowserUtils }); module.exports = __toCommonJS(src_exports); // src/browser-utils.ts var import_dom_accessibility_api2 = require("dom-accessibility-api"); var import_finder2 = require("@medv/finder"); // src/grounding/screenshot.ts async function getPageBrightness(screenshot) { if (!screenshot) { return 100; } const img = document.createElement("img"); return new Promise((resolve, reject) => { img.onload = () => { const width = img.naturalWidth; const height = img.naturalHeight; const canvas = document.createElement("canvas"); const ctx = canvas.getContext("2d"); canvas.width = width; canvas.height = height; ctx.drawImage(img, 0, 0, width, height); const pixels = ctx.getImageData(0, 0, width, height); const count = pixels.data.length; let brightnessTotal = 0; for (let offset = 0; offset < count; offset += 4) { const r = pixels.data[offset]; const g = pixels.data[offset + 1]; const b = pixels.data[offset + 2]; const br = 0.3 * r + 0.59 * g + 0.11 * b; brightnessTotal += br / 255 * 100; } resolve(Math.floor(brightnessTotal / width / height)); }; img.onerror = reject; img.src = screenshot; }); } function isDark(brightness) { return brightness < 45; } function randomBrightness(pageBrightness) { let lower; let upper; if (isDark(pageBrightness)) { lower = 100 - pageBrightness * 2; upper = 100; } else { lower = 0; upper = 100 - pageBrightness / 2; } return Math.trunc(Math.random() * (upper - lower) + lower); } // src/grounding/scroll.ts function getScrollableParent(el) { while (el) { if (isScrollable(el)) { return el; } el = el.parentElement; } return null; } function getScrollTop() { const doc = document.documentElement; return (window.scrollY || doc.scrollTop) - (doc.clientTop || 0); } function isScrollable(el = document.documentElement) { const { overflowY } = getComputedStyle(el); const canScroll = el === document.documentElement || overflowY === "scroll" || overflowY === "auto"; return canScroll && el.scrollTop + el.clientHeight < el.scrollHeight; } function scrollPageDown() { const scrollTop = getScrollTop(); window.scrollTo(window.scrollX, scrollTop + window.innerHeight); } function isScrollablePage() { return ( // getScrollableParent(document.body) !== null && Math.ceil(getScrollTop() + window.innerHeight) < getPageHeight() ); } function getPageHeight() { const body = document.body; const html = document.documentElement; return Math.max( body.scrollHeight, body.offsetHeight, html.clientHeight, html.scrollHeight, html.offsetHeight ); } // src/grounding/constants.ts var attrsToKeepRegex = /value|type|href|title|alt|aria-(?!label|labelledby|description|describedby)|placeholder|disabled/; var a11yAttrs = [ "aria-label", "aria-description", "alt", "title", "value", "placeholder" ]; var a11ySelectors = a11yAttrs.map((attr) => `[${attr}]`).join(","); var markerAttr = "data-marker-id"; var bboxClassName = "lc-marker"; var defaultSelector = 'a, button, input, textarea, summary, select, [role="button"], [role="tab"], [role="link"], [role="checkbox"], [role="menuitem"], [role="menuitemcheckbox"], [role="menuitemradio"], [role="radio"], [role="combobox"], [role="option"], [role="searchbox"], [role="textbox"], [contenteditable]'; // src/grounding/annotation.ts function ensureStyle() { if (document.querySelector("style#lc-style")) { return; } const style = document.createElement("style"); style.textContent = ` .lc-marker { position: absolute; border: 2px solid var(--bg-color); pointer-events: none; box-sizing: border-box; z-index: 1073741825; } .lc-marker::before { content: attr(data-element-id); position: absolute; /* bottom: 100%; */ /* left: -1px; */ top: 0; left: 0; padding: 0px 2px; color: var(--text-color); background-color: var(--bg-color); font-size: 12px; } `; document.head.append(style); } function getZindex(el) { let zIndex = -Infinity; while (el && el !== document.body) { const z = parseInt(window.getComputedStyle(el).zIndex, 10); if (z > zIndex) { zIndex = z; } el = el.parentElement; } return isFinite(zIndex) ? zIndex.toString() : "auto"; } function markElement({ el, id, pageBrightness, bgColor, textColor }) { ensureStyle(); const scrollTop = getScrollTop(); const rects = el.getClientRects(); const brightness = randomBrightness(pageBrightness); const zIndex = getZindex(el); if (!bgColor) { bgColor = `hsl(${Math.random() * 360 | 0}, ${Math.random() * 100 | 0}%, ${brightness}%)`; } if (!textColor) { textColor = isDark(pageBrightness) ? "#000" : "#fff"; } for (const rect of rects) { if (rect.width === 0 || rect.height === 0) { continue; } const marker = document.createElement("div"); marker.className = bboxClassName; marker.style.top = scrollTop + rect.top + "px"; marker.style.left = rect.left + "px"; marker.style.width = rect.width + "px"; marker.style.height = rect.height + "px"; marker.style.zIndex = zIndex; el.setAttribute(markerAttr, id.toString()); marker.setAttribute("data-element-id", id.toString()); marker.style.setProperty("--bg-color", bgColor); marker.style.setProperty("--text-color", textColor); document.body.appendChild(marker); } } async function addBboxes(elements, screenshot) { const pageBrightness = await getPageBrightness(screenshot); elements.forEach((el, i) => { markElement({ id: i, el, pageBrightness }); }); return elements; } function clearBboxes() { document.querySelector("#lc-mask")?.remove(); [...document.querySelectorAll(`.${bboxClassName}`)].forEach( (el) => el.remove() ); [...document.querySelectorAll(`[${markerAttr}]`)].forEach((el) => { el.removeAttribute(markerAttr); }); } // src/grounding/dom.ts var import_dom_accessibility_api = require("dom-accessibility-api"); // src/shared/delay.ts async function delay(ms) { await new Promise((res) => setTimeout(res, ms)); } // src/shared/dom.ts function getNearestElementOfType(el, assertFn) { while (el && el !== document.body) { if (assertFn(el)) { return el; } el = el.parentElement; } return null; } function getInnerMostElements(elements) { const result = new Set(elements); for (const outer of result) { for (const inner of result) { if (outer !== inner && outer.contains(inner)) { result.delete(outer); } } } return result; } // src/grounding/dom.ts function isContenteditable(el) { const editable = el.getAttribute("contenteditable"); return editable === "" || editable === "true" || editable === "plaintext-only"; } function hasA11yInfo(el) { return !!((0, import_dom_accessibility_api.computeAccessibleName)(el) || (0, import_dom_accessibility_api.computeAccessibleDescription)(el)); } function getClickableElements() { const result = /* @__PURE__ */ new Set(); const elements = document.querySelectorAll("*"); for (const el of elements) { if (result.has(el)) { continue; } const styles = getComputedStyle(el); if ( // el instanceof HTMLElement && styles.cursor === "pointer" && styles.pointerEvents !== "none" && hasA11yInfo(el) ) { result.add( el instanceof SVGElement ? getNearestElementOfType( el, (el2) => el2.tagName === "svg" ) || el : el ); } } return result; } function isFormComponent(el) { const { tagName } = el; return tagName === "INPUT" || tagName === "TEXTAREA" || tagName === "SELECT" || tagName === "OPTION"; } function isVisibleForUser(el) { const rects = el.getClientRects(); const slices = 4; for (const rect of rects) { for (let i = 0; i <= slices; i++) { const x = rect.left + rect.width * i / slices; const y = rect.top + rect.height * i / slices; const topEl = document.elementFromPoint(x, y); if (el.contains(topEl)) { return true; } } } return false; } function getInteractiveElements(selector = defaultSelector, fullPage = false) { const { innerWidth, innerHeight } = window; const clickableElements = getInnerMostElements(getClickableElements()); const baseElements = new Set( [...document.querySelectorAll(selector)].filter((el) => { if (el.tagName === "SUMMARY") { return true; } const detailsEl = getNearestElementOfType( el, (el2) => el2.tagName === "DETAILS" ); return !detailsEl || detailsEl.open; }) ); for (const el of baseElements) { if (el.children.length === 1 && baseElements.has(el.firstElementChild)) { baseElements.delete(el); } } for (const el of baseElements) { for (const clickable of clickableElements) { if (clickable.contains(el) || el.contains(clickable)) { clickableElements.delete(clickable); } } } const targets = [.../* @__PURE__ */ new Set([...baseElements, ...clickableElements])].filter((el) => !(0, import_dom_accessibility_api.isDisabled)(el) && !(0, import_dom_accessibility_api.isInaccessible)(el)).filter((el) => { if (el.matches("textarea.monaco-mouse-cursor-text")) { return true; } const bounding = el.getBoundingClientRect(); if (bounding.width === 0 || bounding.height === 0) { return false; } if (fullPage) { return true; } const scrollableParent = getScrollableParent(el); if (scrollableParent && scrollableParent !== document.documentElement && scrollableParent !== document.body) { return true; } const isInsideViewport = bounding.top < innerHeight && bounding.bottom > 0 && bounding.left < innerWidth && bounding.right > 0; if (!isInsideViewport) { return false; } return isVisibleForUser(el); }).map((el, i) => { el.setAttribute(markerAttr, i.toString()); return el; }); return targets; } // src/grounding/contentful-elements.ts function isContentlessEl(el) { if (/^body|head|html|script|noscript|style|select|form|iframe$/i.test( el.tagName )) { return true; } const rect = el.getBoundingClientRect(); return rect.width === 0 || rect.height === 0; } function getMostContentfulElements(topN = 3) { const childrenCountMap = /* @__PURE__ */ new Map(); [...document.body.querySelectorAll("*")].forEach((el) => { if (isContentlessEl(el)) { return; } const tagCount = /* @__PURE__ */ new Map(); for (const child of el.children) { if (isContentlessEl(child)) { continue; } const tagName = child.tagName; const textContent = child.textContent; if (child.querySelector("a[href], img[src]") || textContent && textContent.length > 20) { tagCount.set(tagName, (tagCount.get(tagName) || 0) + 1); } } const mostCommonTag = [...tagCount.entries()].sort( (a, b) => b[1] - a[1] )[0]; childrenCountMap.set( el, mostCommonTag ? { tagName: mostCommonTag[0], count: mostCommonTag[1] } : { tagName: null, count: 0 } ); }); return [...childrenCountMap.entries()].sort((a, b) => b[1].count - a[1].count).slice(0, topN); } // src/grounding/selector.ts var selector_exports = {}; __export(selector_exports, { expandAnchorFrom: () => expandAnchorFrom, getClosestElement: () => getClosestElement, getCommonAncestor: () => getCommonAncestor, getCommonItemsAndAncestor: () => getCommonItemsAndAncestor, getCommonSelector: () => getCommonSelector, isChildOf: () => isChildOf, queryAncestor: () => queryAncestor }); var import_finder = require("@medv/finder"); function queryAncestor(el, selector) { let ancestor = el; while (ancestor) { if (ancestor.matches(selector)) { return ancestor; } ancestor = ancestor.parentElement; } return null; } function isChildOf(el, ancestorSelector) { return el.matches(`${ancestorSelector}, ${ancestorSelector} *`); } function getClosestElement(el, selector) { return el.closest(selector) || el.querySelector(selector); } function getCommonAncestor(...els) { if (els.length === 0) { return null; } if (els.length === 1) { return els[0].parentElement; } const [el0, ...restEls] = els; const ancestors = /* @__PURE__ */ new Set(); let ancestor = el0; while (ancestor) { ancestors.add(ancestor); ancestor = ancestor.parentElement; } let candidate = null; for (const el of restEls) { let commonAncestor = null; ancestor = el; while (ancestor) { if (ancestors.has(ancestor)) { commonAncestor = ancestor; break; } ancestor = ancestor.parentElement; } if (!commonAncestor) { return null; } if (!candidate || commonAncestor.contains(candidate)) { candidate = commonAncestor; } } return candidate; } function getCommonSelector(root, ...els) { if (els.length === 0) { return null; } const [el0, ...restEls] = els; const selectors = []; for (const cls of el0.classList) { const matched = restEls.every((el) => el.classList.contains(cls)); if (matched) { selectors.push(`.${CSS.escape(cls)}`); } } for (const attr of el0.attributes) { if (/^class|id|style|src|alt|href$/.test(attr.name)) { continue; } const matched = restEls.every( (el) => el.getAttribute(attr.name) === attr.value ); if (matched) { selectors.push(`[${CSS.escape(attr.name)}="${CSS.escape(attr.value)}"]`); } } if (selectors.length === 0) { const allSameTag = restEls.every((el) => el.tagName === el0.tagName); if (!allSameTag) { return null; } const tagNameSelector = el0.tagName.toLowerCase(); const parents = [el0.parentElement, ...restEls.map((el) => el.parentElement)]; if (parents.some((p) => !p)) { return null; } if (parents.some((p) => p === root)) { const p0 = parents[0]; if (parents.every((p) => p === p0)) { return "> " + tagNameSelector; } return tagNameSelector; } const parentSelector = getCommonSelector(root, ...parents); if (!parentSelector) { return null; } return parentSelector + " > " + tagNameSelector; } let lastMatchedCount = 0; for (let i = selectors.length - 1; i >= 0; i--) { const selector = selectors.join(""); const matched = root.querySelectorAll("& > " + selector); if (!lastMatchedCount) { lastMatchedCount = matched.length; } if (!matched.length || matched.length !== lastMatchedCount) { return "> " + selectors.slice(0, i + 2).join(""); } } return "> " + selectors[0]; } function getCommonItemsAndAncestor(...els) { const ancestor = getCommonAncestor(...els); if (!ancestor) { return null; } const ancestorSelector = (0, import_finder.finder)(ancestor); const commonSelector = getCommonSelector(ancestor, ...els) || "> *"; return { items: `${ancestorSelector} ${commonSelector}`, ancestor: ancestorSelector }; } function expandAnchorFrom(fromElem) { let anchor = fromElem; let maxMatches = 0; let bestItemSelector = null; let bestMatches = null; while (anchor) { const selectors = getCommonItemsAndAncestor(anchor); const currentAnchor = anchor; anchor = anchor.parentElement; if (selectors?.items) { const lastSelector = selectors.items.split(" ").at(-1); if (!lastSelector || !(lastSelector.startsWith(".") || lastSelector.startsWith("["))) { continue; } const matches = document.querySelectorAll(selectors.items); const outerMostMatches = new Set(matches); if (currentAnchor !== fromElem) { const originalElemSelector = (0, import_finder.finder)(fromElem, { root: currentAnchor, idName() { return false; } }); for (const el of outerMostMatches) { if (!el.querySelector(originalElemSelector)) { outerMostMatches.delete(el); } } } for (const el1 of matches) { for (const el2 of outerMostMatches) { if (el1 !== el2 && el1.contains(el2)) { outerMostMatches.delete(el2); } } } if (outerMostMatches.size > maxMatches) { maxMatches = outerMostMatches.size; bestItemSelector = selectors.items; bestMatches = outerMostMatches; } } } if (bestItemSelector && bestMatches && bestMatches.size > 1) { const matches = [...bestMatches]; const index = matches.findIndex((el) => el.contains(fromElem)); return [matches[index], matches[(index + 1) % matches.length]]; } return null; } // src/execution/observer.ts var Observer = class { #stablePromise; #hasChange = false; #observer; #overallTimer; constructor(maxTimeout = 5e3) { this.#stablePromise = new Promise((resolve) => { let timer; const done = () => { resolve(); clearTimeout(timer); clearTimeout(this.#overallTimer); observer.disconnect(); window.removeEventListener("scroll", resolveDelayed); }; const resolveDelayed = () => { clearTimeout(timer); timer = window.setTimeout(() => { done(); }, 3e3); }; window.addEventListener("scroll", resolveDelayed); this.#overallTimer = window.setTimeout(() => { done(); }, maxTimeout); const observer = new MutationObserver((mutations) => { for (const m of mutations) { if (m.target !== document.body) { this.#hasChange = true; break; } } if (this.#hasChange) { resolveDelayed(); } }); observer.observe(document.body, { subtree: true, childList: true }); this.#observer = observer; }); } async domStable() { await this.#stablePromise; } domChanged() { return this.#hasChange; } disconnect() { this.#observer.disconnect(); clearTimeout(this.#overallTimer); } }; // src/execution/synthetic-events.ts function setValue(el, value) { const proto = Object.getPrototypeOf(el); const setter = Object.getOwnPropertyDescriptor(proto, "value")?.set; if (setter) { setter.call(el, value); } else { el.value = value; } } function focusInputElement(el) { el.click(); el.focus(); el.dispatchEvent( new Event("focusin", { bubbles: true }) ); } async function syntheticClick(el) { const bounding = el.getBoundingClientRect(); const mouseEventInitial = { bubbles: true, which: 1, clientX: bounding.x + bounding.width / 2, clientY: bounding.y + bounding.height / 2 }; el.dispatchEvent(new MouseEvent("mousedown", mouseEventInitial)); el.dispatchEvent(new PointerEvent("pointerdown", mouseEventInitial)); if (el instanceof HTMLElement) { el.click(); } else { el.dispatchEvent(new MouseEvent("click", mouseEventInitial)); } el.dispatchEvent(new MouseEvent("mouseup", mouseEventInitial)); el.dispatchEvent(new PointerEvent("pointerup", mouseEventInitial)); } async function syntheticInput(el, value) { setValue(el, value); focusInputElement(el); await delay(300); setValue(el, value); el.dispatchEvent( new Event("input", { bubbles: true }) ); el.dispatchEvent( new Event("change", { bubbles: true }) ); } async function syntheticInputOnContenteditable(el, value) { focusInputElement(el); await delay(300); const selection = window.getSelection(); selection?.removeAllRanges(); const range = document.createRange(); range.selectNodeContents(el); selection?.addRange(range); document.execCommand("delete"); await delay(300); document.execCommand("insertText", false, value); } async function syntheticEnter(el) { const form = getNearestElementOfType( el, (el2) => el2.tagName === "FORM" ); if (form) { if (form.requestSubmit) { form.requestSubmit(); } else { form.submit(); } } focusInputElement(el); await delay(300); const keyboardEventInitial = { key: "Enter", code: "Enter", keyCode: 13, charCode: 13, bubbles: true }; el.dispatchEvent(new KeyboardEvent("keydown", keyboardEventInitial)); el.dispatchEvent(new KeyboardEvent("keyup", keyboardEventInitial)); el.dispatchEvent(new KeyboardEvent("keypress", keyboardEventInitial)); } // src/browser-utils.ts var BrowserUtils = class { constructor(selector = defaultSelector) { this.selector = selector; } #prevElements = []; #observer = null; static getUniqueSelector = import_finder2.finder; static selectorUtils = selector_exports; getUniqueSelector = import_finder2.finder; selectorUtils = selector_exports; getElement(id) { const el = this.#prevElements.at(Number(id)); if (el?.isConnected) { return el; } return document.querySelector(`[data-marker-id="${id}"]`); } addBboxes(fullPage) { const interactiveElements = getInteractiveElements(this.selector, fullPage); addBboxes(interactiveElements); } clearBboxes() { clearBboxes(); } elementToJSON(el) { const elemJSON = { id: el.getAttribute(markerAttr) || "", tag: el.tagName.toLowerCase(), role: (0, import_dom_accessibility_api2.getRole)(el), accessibleName: (0, import_dom_accessibility_api2.computeAccessibleName)(el), accessibleDescription: (0, import_dom_accessibility_api2.computeAccessibleDescription)(el), attributes: {} }; if (el.tagName === "SELECT") { elemJSON.options = [...el.querySelectorAll("option")].map( (opt) => opt.value ); } for (const attr of [...el.attributes]) { if (attrsToKeepRegex.test(attr.name)) { elemJSON.attributes[attr.name] = attr.value; } } if (isFormComponent(el)) { const value = el.value; if (value && el.getAttribute("type") !== "password") { elemJSON.attributes.value = value; } } const href = el.getAttribute("href"); if (href) { if (!elemJSON.accessibleName && !elemJSON.accessibleDescription) { elemJSON.attributes.href = href.slice(0, 100); } else { delete elemJSON.attributes.href; } } return elemJSON; } async snapshot(screenshot, fullPage) { this.clearBboxes(); const addedIDs = /* @__PURE__ */ new Set(); const prevElemSet = new Set(this.#prevElements); const interactiveElements = getInteractiveElements(this.selector, fullPage); const pageBrightness = await getPageBrightness(screenshot); const elementsAsJSON = interactiveElements.map((el, i) => { markElement({ id: i, el, pageBrightness }); const markerId = el.getAttribute(markerAttr); if (markerId === null) { throw new Error(`Unable to find markerId for element: ${el.outerHTML}`); } if (!prevElemSet.has(el)) { addedIDs.add(markerId); } return this.elementToJSON(el); }); this.#prevElements = interactiveElements; return { interactiveElements, elementsAsJSON, addedIDs: [...addedIDs] }; } async getMostContentfulElements(screenshot, topN = 3) { this.clearBboxes(); const pageBrightness = await getPageBrightness(screenshot); const mostContentful = getMostContentfulElements(topN); let id = 0; this.#prevElements = []; return mostContentful.flatMap(([el, mostFrequentChildren], i) => { const brightness = randomBrightness(pageBrightness); const bgColor = `hsl(${Math.random() * 360 | 0}, ${Math.random() * 100 | 0}%, ${brightness}%)`; const textColor = isDark(pageBrightness) ? "#000" : "#fff"; return [...el.children].filter((child) => { return child.tagName === mostFrequentChildren.tagName && !isContentlessEl(child); }).map((child) => { markElement({ el: child, id, pageBrightness, bgColor, textColor }); this.#prevElements.push(child); const data = { id, groupId: i, html: child.outerHTML }; id++; return data; }); }); } initObserver(maxTimeout) { this.#observer = new Observer(maxTimeout); } isScrollable() { return isScrollablePage(); } async stable() { await this.#observer?.domStable(); } async scrollPageDown() { return scrollPageDown(); } async click(el) { return syntheticClick(el); } async fill(el, value) { if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { return syntheticInput(el, value); } if (isContenteditable(el)) { return syntheticInputOnContenteditable(el, value); } throw new Error("Unable to fill in a non-form element"); } async select(el, value) { if (!(el instanceof HTMLSelectElement)) { throw new Error("Unable to select a non-select element"); } return syntheticInput(el, value); } async enter(el) { if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { return syntheticEnter(el); } throw new Error("Unable to press enter on a non-form element"); } };