# UIBridgeJS MVP Implementation Plan ## MVP Scope: Core Features Only ### Features to Implement 1. **DOM Element Finding** - Multiple selector strategies 2. **Click Command** - Synthetic click events 3. **Screenshot Command** - Full viewport capture 4. **Command Discovery Interface (CDI)** - Auto-generated command list --- ## Project Structure ``` uibridge-js/ ├── src/ │ ├── core/ │ │ ├── UIBridge.js # Main class │ │ ├── CommandRegistry.js # Command registration │ │ ├── SelectorEngine.js # DOM finding logic │ │ └── EventSynthesizer.js # Native event generation │ ├── commands/ │ │ ├── click.js │ │ └── screenshot.js │ ├── discovery/ │ │ ├── CDIGenerator.js # Generates discovery files │ │ └── templates/ │ │ └── commands.md.template │ └── index.js ├── dist/ │ └── uibridge.min.js ├── test/ │ └── sveltekit-app/ # Test SvelteKit app └── package.json ``` --- ## Implementation Plan ### Phase 1: Core Architecture (Days 1-2) ✅ COMPLETED #### 1.1 Main UIBridge Class ✅ ```javascript // src/core/UIBridge.js class UIBridge { constructor(config = {}) { this.config = { debug: false, allowedOrigins: ['*'], commands: ['click', 'screenshot', 'discover'], ...config }; this.registry = new CommandRegistry(); this.selectorEngine = new SelectorEngine(); this._isInitialized = false; } init() { if (this._isInitialized) return; // Register core commands this.registry.register('click', clickCommand); this.registry.register('screenshot', screenshotCommand); // Setup discovery endpoint this._setupDiscovery(); // Expose global API window.UIBridge = this; window.__uibridge__ = { execute: this.execute.bind(this), discover: this.discover.bind(this) }; this._isInitialized = true; this._log('UIBridge initialized'); } async execute(commandName, ...args) { const command = this.registry.get(commandName); if (!command) { throw new Error(`Unknown command: ${commandName}`); } this._log(`Executing: ${commandName}`, args); return await command.execute(this, ...args); } findElement(selector) { return this.selectorEngine.find(selector); } discover() { return this.registry.getAll().map(cmd => ({ name: cmd.name, description: cmd.description, parameters: cmd.parameters })); } _log(...args) { if (this.config.debug) { console.log('[UIBridge]', ...args); } } } ``` #### 1.2 Selector Engine ✅ ```javascript // src/core/SelectorEngine.js class SelectorEngine { constructor() { this.strategies = new Map(); this._setupDefaultStrategies(); } _setupDefaultStrategies() { // CSS Selector this.strategies.set('css', (selector) => { return document.querySelector(selector); }); // XPath this.strategies.set('xpath', (xpath) => { const result = document.evaluate( xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null ); return result.singleNodeValue; }); // Text content this.strategies.set('text', (text) => { const walker = document.createTreeWalker( document.body, NodeFilter.SHOW_TEXT, null, false ); let node; while (node = walker.nextNode()) { if (node.textContent.trim() === text) { return node.parentElement; } } return null; }); // Data-testid this.strategies.set('testId', (id) => { return document.querySelector(`[data-testid="${id}"]`); }); } find(selector) { // String = CSS selector if (typeof selector === 'string') { return this.strategies.get('css')(selector); } // Object selectors if (typeof selector === 'object') { if (selector.xpath) { return this.strategies.get('xpath')(selector.xpath); } if (selector.text) { return this.strategies.get('text')(selector.text); } if (selector.testId) { return this.strategies.get('testId')(selector.testId); } } throw new Error(`Invalid selector: ${JSON.stringify(selector)}`); } findAll(selector) { // Implementation for multiple elements // Similar to find() but returns array } } ``` --- ### Phase 2: Commands Implementation (Days 3-4) ✅ COMPLETED #### 2.1 Click Command ✅ ```javascript // src/commands/click.js export const clickCommand = { name: 'click', description: 'Clicks on an element', parameters: [ { name: 'selector', type: 'Selector', required: true, description: 'Element to click' }, { name: 'options', type: 'ClickOptions', required: false, description: 'Click options' } ], async execute(bridge, selector, options = {}) { const element = bridge.findElement(selector); if (!element) { throw new Error(`Element not found: ${JSON.stringify(selector)}`); } // Default options const opts = { force: false, position: 'center', button: 'left', clickCount: 1, delay: 0, ...options }; // Check visibility unless force is true if (!opts.force) { const isVisible = this._isElementVisible(element); if (!isVisible) { throw new Error('Element is not visible'); } } // Calculate click position const rect = element.getBoundingClientRect(); const position = this._calculatePosition(rect, opts.position); // Create synthetic mouse events const eventInit = { bubbles: true, cancelable: true, view: window, clientX: position.x, clientY: position.y, button: opts.button === 'right' ? 2 : 0, buttons: opts.button === 'right' ? 2 : 1, }; // Dispatch events element.dispatchEvent(new MouseEvent('mousedown', eventInit)); if (opts.delay > 0) { await new Promise(resolve => setTimeout(resolve, opts.delay)); } element.dispatchEvent(new MouseEvent('mouseup', eventInit)); element.dispatchEvent(new MouseEvent('click', eventInit)); // Handle multiple clicks if (opts.clickCount > 1) { for (let i = 1; i < opts.clickCount; i++) { await new Promise(resolve => setTimeout(resolve, 50)); element.dispatchEvent(new MouseEvent('click', eventInit)); } } // Focus element if it's focusable if (this._isFocusable(element)) { element.focus(); } return { success: true, element: { tag: element.tagName.toLowerCase(), text: element.textContent.trim().substring(0, 100) } }; }, _isElementVisible(element) { const rect = element.getBoundingClientRect(); const style = window.getComputedStyle(element); return ( rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && rect.top < window.innerHeight && rect.bottom > 0 && rect.left < window.innerWidth && rect.right > 0 ); }, _calculatePosition(rect, position) { const positions = { center: { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }, topLeft: { x: rect.left, y: rect.top }, topRight: { x: rect.right, y: rect.top }, bottomLeft: { x: rect.left, y: rect.bottom }, bottomRight: { x: rect.right, y: rect.bottom } }; return positions[position] || positions.center; }, _isFocusable(element) { const focusableTags = ['input', 'select', 'textarea', 'button', 'a']; return focusableTags.includes(element.tagName.toLowerCase()) || element.hasAttribute('tabindex'); } }; ``` #### 2.2 Screenshot Command ✅ ```javascript // src/commands/screenshot.js export const screenshotCommand = { name: 'screenshot', description: 'Takes a screenshot of the page or element', parameters: [ { name: 'options', type: 'ScreenshotOptions', required: false, description: 'Screenshot options' } ], async execute(bridge, options = {}) { const opts = { selector: null, format: 'png', quality: 0.92, fullPage: false, ...options }; let targetElement = document.body; // Find specific element if selector provided if (opts.selector) { targetElement = bridge.findElement(opts.selector); if (!targetElement) { throw new Error(`Element not found: ${JSON.stringify(opts.selector)}`); } } // Use html2canvas library (loaded dynamically) await this._loadHtml2Canvas(); const canvas = await window.html2canvas(targetElement, { useCORS: true, allowTaint: false, backgroundColor: null, scale: window.devicePixelRatio || 1, width: opts.fullPage ? document.documentElement.scrollWidth : undefined, height: opts.fullPage ? document.documentElement.scrollHeight : undefined, windowWidth: opts.fullPage ? document.documentElement.scrollWidth : undefined, windowHeight: opts.fullPage ? document.documentElement.scrollHeight : undefined, x: opts.fullPage ? 0 : undefined, y: opts.fullPage ? 0 : undefined }); // Convert to desired format const dataUrl = canvas.toDataURL(`image/${opts.format}`, opts.quality); return { success: true, dataUrl, width: canvas.width, height: canvas.height, format: opts.format }; }, async _loadHtml2Canvas() { if (window.html2canvas) return; return new Promise((resolve, reject) => { const script = document.createElement('script'); script.src = 'https://cdnjs.cloudflare.com/ajax/libs/html2canvas/1.4.1/html2canvas.min.js'; script.onload = resolve; script.onerror = reject; document.head.appendChild(script); }); } }; ``` --- ### Phase 3: Command Discovery Interface (Days 5-6) ✅ COMPLETED #### 3.1 CDI Generator ✅ ```javascript // src/discovery/CDIGenerator.js class CDIGenerator { constructor(registry) { this.registry = registry; } generateMarkdown() { const commands = this.registry.getAll(); const date = new Date().toISOString(); let markdown = `# UIBridge Commands\n\n`; markdown += `Generated: ${date}\n\n`; markdown += `## Available Commands\n\n`; // Summary table markdown += `| Command | Description | Parameters |\n`; markdown += `|---------|-------------|------------|\n`; commands.forEach(cmd => { const params = cmd.parameters.map(p => `${p.name}${p.required ? '' : '?'}` ).join(', '); markdown += `| ${cmd.name} | ${cmd.description} | ${params} |\n`; }); // Detailed documentation markdown += `\n## Command Details\n\n`; commands.forEach(cmd => { markdown += `### ${cmd.name}\n\n`; markdown += `${cmd.description}\n\n`; if (cmd.parameters.length > 0) { markdown += `**Parameters:**\n\n`; cmd.parameters.forEach(param => { markdown += `- \`${param.name}\` (${param.type})${param.required ? ' **required**' : ''}: ${param.description}\n`; }); markdown += '\n'; } // Add examples if available if (cmd.examples) { markdown += `**Examples:**\n\n`; cmd.examples.forEach(example => { markdown += `\`\`\`javascript\n${example}\n\`\`\`\n\n`; }); } }); return markdown; } generateJSON() { const commands = this.registry.getAll(); return { version: '1.0.0', generated: new Date().toISOString(), commands: commands.map(cmd => ({ name: cmd.name, description: cmd.description, parameters: cmd.parameters, examples: cmd.examples || [] })) }; } async saveToFile(format = 'markdown') { const content = format === 'json' ? JSON.stringify(this.generateJSON(), null, 2) : this.generateMarkdown(); const blob = new Blob([content], { type: format === 'json' ? 'application/json' : 'text/markdown' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = `uibridge-commands.${format === 'json' ? 'json' : 'md'}`; a.click(); URL.revokeObjectURL(url); } } ``` #### 3.2 Discovery Setup ```javascript // Addition to UIBridge.js _setupDiscovery() { // HTTP endpoint simulation (for development) if (this.config.enableHttpDiscovery) { // In production, this would be a real HTTP endpoint window.__uibridge_discovery__ = () => { return this.discover(); }; } // Generate CDI file on init if configured if (this.config.generateCDI) { const generator = new CDIGenerator(this.registry); // Auto-save in development if (this.config.debug) { console.log('[UIBridge] CDI Markdown:'); console.log(generator.generateMarkdown()); } } } ``` --- ### Phase 4: SvelteKit Integration & Testing (Days 7-8) ✅ COMPLETED #### 4.1 SvelteKit Test App Structure ``` test/sveltekit-app/ ├── src/ │ ├── app.html │ ├── routes/ │ │ ├── +layout.svelte │ │ ├── +page.svelte │ │ └── test/ │ │ └── +page.svelte │ └── lib/ │ └── UIBridge.svelte ├── static/ │ └── uibridge.js # Built library └── package.json ``` #### 4.2 SvelteKit Integration Component ```svelte ``` #### 4.3 Test Page ```svelte

UIBridge Test Page

Message: {message}

Test Controls

Available Commands:

{#if screenshotUrl}

Screenshot Result:

Screenshot
{/if}
``` #### 4.4 Layout with UIBridge ```svelte ``` --- ### Phase 5: Build & Bundle Configuration (Day 9) ✅ COMPLETED #### 5.1 Build Script ```javascript // build.js import { build } from 'esbuild'; import fs from 'fs'; // Build minified version build({ entryPoints: ['src/index.js'], bundle: true, minify: true, sourcemap: true, outfile: 'dist/uibridge.min.js', format: 'iife', globalName: 'UIBridge', target: ['es2020'], define: { VERSION: '"1.0.0"' } }).then(() => { console.log('✓ Built uibridge.min.js'); // Copy to test app fs.copyFileSync( 'dist/uibridge.min.js', 'test/sveltekit-app/static/uibridge.js' ); console.log('✓ Copied to SvelteKit test app'); }); ``` #### 5.2 Package.json ```json { "name": "uibridge", "version": "0.1.0", "description": "In-app automation framework for web applications", "main": "dist/uibridge.min.js", "scripts": { "build": "node build.js", "dev": "node build.js --watch", "test": "cd test/sveltekit-app && npm run dev", "test:e2e": "playwright test" }, "devDependencies": { "esbuild": "^0.17.0", "@playwright/test": "^1.30.0" }, "keywords": ["automation", "testing", "ai", "web"], "license": "MIT" } ``` --- ### Phase 6: Testing Plan (Day 10) ✅ COMPLETED ### Phase 7: Help System Implementation ✅ COMPLETED #### 7.1 Command Line Help System ✅ ```bash # Comprehensive help for AI agents and developers npm run help # Show all commands and usage npm run help click # Detailed help for click command npm run help screenshot # Detailed help for screenshot command npm run help:examples # Show usage patterns and examples npm run help:tips # Show best practices and troubleshooting ``` #### 7.2 In-Browser Help Command ✅ ```javascript // Get comprehensive help information const help = await uibridge.execute('help'); // Get help for specific command const clickHelp = await uibridge.execute('help', 'click'); const screenshotHelp = await uibridge.execute('help', 'screenshot'); // Alternative syntax const help = await uibridge.execute('--help'); ``` #### 7.3 AI Agent Documentation ✅ - **AGENT_HELP_GUIDE.md** - Comprehensive guide for LLM consumption - **Structured JSON output** - Perfect for programmatic parsing - **Selector strategies** - All supported element finding methods - **Common patterns** - Real-world usage examples - **Error handling** - Troubleshooting and fallback strategies - **Best practices** - Tips for reliable automation #### 7.4 Help System Features ✅ - **Command discovery** - List all available commands - **Parameter documentation** - Types, requirements, descriptions - **Usage examples** - Copy-paste ready code snippets - **Selector reference** - CSS, XPath, text, testId, aria-label, label - **Troubleshooting guide** - Common issues and solutions - **Integration patterns** - Framework-specific examples ### Phase 8: Testing Plan (Day 10) ✅ COMPLETED #### 8.1 Manual Testing Checklist ✅ COMPLETED ```markdown ## UIBridge MVP Testing Checklist ### Basic Functionality ✅ COMPLETED - [x] Library loads without errors - [x] UIBridge initializes correctly - [x] Global API is accessible (window.uibridge) ### Click Command ✅ COMPLETED - [x] Clicks buttons successfully - [x] Click events trigger native handlers - [x] Works with different selector types: - [x] CSS selector - [x] XPath - [x] Text content - [x] data-testid - [x] Handles non-visible elements correctly - [x] Focus is set on focusable elements ### Screenshot Command ✅ COMPLETED - [x] Captures full viewport - [x] Captures specific elements - [x] Returns base64 data URL - [x] Works with different image formats ### Command Discovery ✅ COMPLETED - [x] discover() returns command list - [x] Generated markdown is correct - [x] JSON format is valid ### SvelteKit Integration ✅ COMPLETED - [x] No SSR errors - [x] Loads correctly in browser - [x] Commands work with Svelte components - [x] No conflicts with Svelte reactivity ### AI Help System ✅ COMPLETED - [x] AI-friendly help command implemented - [x] Comprehensive AGENT_HELP_GUIDE.md created - [x] AI-optimized README with automation patterns - [x] Command-line help script (uibridge-help.js) - [x] UIBRIDGE_COMMANDS.md with AI examples - [x] Export issues fixed for SSR compatibility - [x] NPM package ready for AI agent consumption ``` #### 6.2 E2E Test Example ```javascript // test/e2e/basic.spec.js import { test, expect } from '@playwright/test'; test.describe('UIBridge MVP', () => { test.beforeEach(async ({ page }) => { await page.goto('http://localhost:5173/test'); await page.waitForFunction(() => window.uibridge); }); test('click command works', async ({ page }) => { // Execute click via UIBridge await page.evaluate(async () => { await window.uibridge.execute('click', '#test-button'); }); // Verify result const message = await page.textContent('p'); expect(message).toContain('Button clicked via UIBridge!'); }); test('screenshot command works', async ({ page }) => { const result = await page.evaluate(async () => { return await window.uibridge.execute('screenshot'); }); expect(result.success).toBe(true); expect(result.dataUrl).toMatch(/^data:image\/png;base64,/); }); test('discover returns commands', async ({ page }) => { const commands = await page.evaluate(() => { return window.uibridge.discover(); }); expect(commands).toHaveLength(2); expect(commands.map(c => c.name)).toContain('click'); expect(commands.map(c => c.name)).toContain('screenshot'); }); }); ``` --- ## Development Timeline | Day | Tasks | Deliverable | |-----|-------|-------------| | 1-2 | Core architecture | UIBridge class, Registry, SelectorEngine | | 3-4 | Commands implementation | Click & Screenshot commands | | 5-6 | CDI implementation | Discovery interface & file generation | | 7-8 | SvelteKit integration | Test app with working examples | | 9 | Build & bundle | Minified distribution file | | 10 | Testing & refinement | Passing test suite | --- ## Next Steps After MVP 1. **Add more selector strategies** (label, aria-label, placeholder) 2. **Implement wait/waitFor commands** for async operations 3. **Add HTTP endpoint** for external agent communication 4. **Create browser extension** for command recording 5. **Add more commands** (type, navigate, scroll) 6. **Improve error messages** with suggestions 7. **Add telemetry** for command success tracking 8. **Create interactive documentation** site --- ## Quick Start Commands ```bash # Create project mkdir uibridge-js && cd uibridge-js npm init -y # Install dependencies npm install -D esbuild @playwright/test # Create directory structure mkdir -p src/{core,commands,discovery} test/sveltekit-app dist # Build library npm run build # Start test app npm run test # Run E2E tests npm run test:e2e ``` This MVP focuses on the absolute essentials while maintaining a clean architecture that can be extended later. The implementation is pragmatic and ready to code!