/** * Internal tools for PageAgent. * @note Adapted from browser-use */ import zod, { type z } from 'zod' import type { PageAgent } from '@/PageAgent' import { clickElement, getElementByIndex, getSystemInfo, inputTextElement, scrollHorizontally, scrollVertically, selectOptionElement, waitFor, } from './actions' // debug import * as utils from './actions' // @ts-expect-error debug only window.utils = utils /** * Internal tool definition that has access to PageAgent `this` context */ export interface PageAgentTool { // name: string description: string inputSchema: z.ZodType execute: (this: PageAgent, args: TParams) => Promise } export function tool(options: PageAgentTool): PageAgentTool { return options } /** * Internal tools for PageAgent. * Note: Using any to allow different parameter types for each tool */ export const tools = new Map() // tools.set( // 'get_current_html', // tool({ // description: 'Get the current (updated) simplified HTML of the page', // inputSchema: zod.object({}), // execute: function (this: PageAgent) { // this.updateTree() // return this.simplifiedHTML // }, // }) // ) tools.set( 'done', tool({ description: 'Complete task - provide a summary of results for the user. Set success=True if task completed successfully, false otherwise. Text should be your response to the user summarizing results.', inputSchema: zod.object({ text: zod.string(), success: zod.boolean().default(true), }), execute: async function (this: PageAgent, input) { // @note main loop will handle this one // this.onDone(input.text, input.success) return Promise.resolve('Task completed') }, }) ) tools.set( 'wait', tool({ description: 'Wait for x seconds. default 1s (max 10 seconds, min 1 second). This can be used to wait until the page or data is fully loaded.', inputSchema: zod.object({ seconds: zod.number().min(1).max(10).default(1), }), execute: async function (this: PageAgent, input) { const lastTimeUpdate = this.lastTimeUpdate const actualWaitTime = Math.max(0, input.seconds - (Date.now() - lastTimeUpdate) / 1000) console.log(`actualWaitTime: ${actualWaitTime} seconds`) await waitFor(actualWaitTime) return `✅ Waited for ${input.seconds} seconds.` + (await getSystemInfo()) }, }) ) tools.set( 'ask_user', tool({ description: 'Ask the user a question and wait for their answer. Use this if you need more information or clarification.', inputSchema: zod.object({ question: zod.string(), }), execute: async function (this: PageAgent, input) { const answer = await this.panel.askUser(input.question) return `✅ Received user answer: ${answer}` + (await getSystemInfo()) }, }) ) tools.set( 'click_element_by_index', tool({ description: 'Click element by index', inputSchema: zod.object({ index: zod.int().min(0), }), execute: async function (this: PageAgent, input) { const element = getElementByIndex(this, input.index) const elemText = this.elementTextMap.get(input.index) await clickElement(element) // @workaround: Handle links that open in new tabs if (element instanceof HTMLAnchorElement && element.target === '_blank') { return `⚠️ Clicked link that opens in a new tab (${elemText ? elemText : input.index}). You are not capable of reading new tabs.` } return `✅ Clicked element (${elemText ? elemText : input.index}).` + (await getSystemInfo()) }, }) ) tools.set( 'input_text', tool({ description: 'Click and input text into a input interactive element', inputSchema: zod.object({ index: zod.int().min(0), text: zod.string(), }), execute: async function (this: PageAgent, input) { const element = getElementByIndex(this, input.index) const elemText = this.elementTextMap.get(input.index) await inputTextElement(element, input.text) return ( `✅ Input text (${input.text}) into element (${elemText ? elemText : input.index}).` + (await getSystemInfo()) ) }, }) ) tools.set( 'select_dropdown_option', tool({ description: 'Select dropdown option for interactive element index by the text of the option you want to select', inputSchema: zod.object({ index: zod.int().min(0), text: zod.string(), }), execute: async function (this: PageAgent, input) { const element = getElementByIndex(this, input.index) const elemText = this.elementTextMap.get(input.index) await selectOptionElement(element as HTMLSelectElement, input.text) return ( `✅ Selected option (${input.text}) in element (${elemText ? elemText : input.index}).` + (await getSystemInfo()) ) }, }) ) /** * @note Reference from browser-use */ tools.set( 'scroll', tool({ description: 'Scroll the page by specified number of pages (set down=True to scroll down, down=False to scroll up, num_pages=number of pages to scroll like 0.5 for half page, 1.0 for one page, etc.). Optional index parameter to scroll within a specific element or its scroll container (works well for dropdowns and custom UI components). Optional pixels parameter to scroll by a specific number of pixels instead of pages.', inputSchema: zod.object({ down: zod.boolean().default(true), num_pages: zod.number().min(0).max(10).optional().default(0.1), pixels: zod.number().int().min(0).optional(), index: zod.number().int().min(0).optional(), }), execute: async function (this: PageAgent, input) { const { down, num_pages, index, pixels } = input const scroll_amount = pixels ? pixels : num_pages * (down ? 1 : -1) * window.innerHeight const element = index !== undefined ? getElementByIndex(this, index) : null return (await scrollVertically(down, scroll_amount, element)) + (await getSystemInfo()) }, }) ) tools.set( 'scroll_horizontally', tool({ description: 'Scroll the page or element horizontally (set right=True to scroll right, right=False to scroll left, pixels=number of pixels to scroll). Optional index parameter to scroll within a specific element or its scroll container (works well for wide tables).', inputSchema: zod.object({ right: zod.boolean().default(true), pixels: zod.number().int().min(0), index: zod.number().int().min(0).optional(), }), execute: async function (this: PageAgent, input) { const { right, pixels, index } = input const scroll_amount = pixels * (right ? 1 : -1) const element = index !== undefined ? getElementByIndex(this, index) : null return (await scrollHorizontally(right, scroll_amount, element)) + (await getSystemInfo()) }, }) ) tools.set( 'execute_javascript', tool({ description: 'Execute JavaScript code on the current page. Supports async/await syntax. Use with caution!', inputSchema: zod.object({ script: zod.string(), }), execute: async function (this: PageAgent, input) { try { // Wrap script in async function to support await const asyncFunction = eval(`(async () => { ${input.script} })`) const result = await asyncFunction() return `✅ Executed JavaScript. Result: ${result}` + (await getSystemInfo()) } catch (error) { return `❌ Error executing JavaScript: ${error}` + (await getSystemInfo()) } }, }) ) // @todo get_dropdown_options // @todo select_dropdown_option // @todo send_keys // @todo upload_file // @todo go_back // @todo extract_structured_data