From 5250bf1e3509dddb3107ba702875463a89edb5a0 Mon Sep 17 00:00:00 2001 From: yu <1919727607@qq.com> Date: Thu, 5 Nov 2026 10:07:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=83=E7=B4=A0=E6=8B=BE=E5=8F=96=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E9=A1=B5=E9=9D=A2=E8=B7=B3=E8=BD=AC=E5=90=8E=E7=BB=A7?= =?UTF-8?q?=E7=BB=AD=E6=8B=BE=E5=8F=96;=20=E6=81=A2=E5=A4=8D=20ws=20?= =?UTF-8?q?=E6=A0=B9=E4=BE=9D=E8=B5=96=E5=A3=B0=E6=98=8E=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pickElement 重构: 拾取脚本经 Page.addScriptToEvaluateOnNewDocument 随新文档自动重注入, 主进程轮询 __rpaPickResult, 用户拾取期间可翻页/搜索/过滑块, 页面跳转不再使拾取失效 - 恢复 package.json ws 根依赖与 electron.vite.config include:[ws] (合并时被覆盖导致 dev 启动报 bufferutil 解析错误) --- package-lock.json | 1 + package.json | 2 +- src/main/browser/session.ts | 503 +++++++++++++++++++----------------- 3 files changed, 270 insertions(+), 236 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6d01e86..c2dcfe9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "react-dom": "^18.3.1", "sm-crypto": "^0.5.8", "tesseract.js": "^7.0.0", + "ws": "^8.18.0", "zustand": "^5.0.2" }, "devDependencies": { diff --git a/package.json b/package.json index 8bc97fc..570d77f 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "react-dom": "^18.3.1", "sm-crypto": "^0.5.8", "tesseract.js": "^7.0.0", - "ws": "^8.22.0", + "ws": "^8.18.0", "zustand": "^5.0.2" }, "devDependencies": { diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 34bbb94..db964ea 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -1153,7 +1153,8 @@ export class BrowserSession implements BrowserLike { } async pickElement(timeoutMs = 90000): Promise { - // 拾取脚本经 Runtime.evaluate 注入并等待用户点击，内置/外部浏览器行为一致 + // 拾取脚本注入当前文档，并经 Page.addScriptToEvaluateOnNewDocument 随页面跳转自动重注入； + // 主进程轮询 window.__rpaPickResult——用户在拾取期间可随意翻页/搜索/过滑块，停在哪就能拾哪。 const url = await this.currentUrl(); if (!url || url === "about:blank") { throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」，再拾取元素" }); @@ -1162,241 +1163,37 @@ export class BrowserSession implements BrowserLike { this.win.show(); this.win.focus(); } - const result = await this.evaluate(`(() => new Promise((resolve, reject) => { - const timeout = ${JSON.stringify(timeoutMs)}; - if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel(); - - let host = null; - let root = null; - let box = null; - let badge = null; - let target = null; - let locked = false; - let chain = []; - let chainIndex = 0; - let selected = null; - let timer = 0; - let finished = false; - - const cssEscape = (value) => { - if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value)); - return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char); - }; - const isUnique = (candidate) => { - try { return document.querySelectorAll(candidate).length === 1; } catch { return false; } - }; - const cssPath = (node) => { - const localPath = (start, root) => { - const stable = (item) => { - if (item.id) { - const candidate = '#' + cssEscape(item.id); - try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} - } - const testId = item.getAttribute('data-testid'); - if (testId) { - const candidate = '[data-testid="' + cssEscape(testId) + '"]'; - try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} - } - return null; - }; - const parts = []; - let current = start; - while (current && current.nodeType === 1) { - const stableCandidate = stable(current); - if (stableCandidate) { - parts.unshift(stableCandidate); - return parts.join(' > '); - } - let part = current.tagName.toLowerCase(); - const parent = current.parentElement; - if (parent) { - const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); - if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; - } - parts.unshift(part); - current = parent; - } - return parts.join(' > '); - }; - const trees = []; - let current = node; - while (current && current.nodeType === 1) { - const root = current.getRootNode(); - trees.unshift(localPath(current, root)); - current = root instanceof ShadowRoot ? root.host : null; + const inject = () => + this.evaluate(`(() => { try { delete window.__rpaPickResult; ${PICKER_BOOTSTRAP_SOURCE} } catch {} })()`); + await this.cdp("Page.enable").catch(() => {}); + const registered = await this.cdp<{ identifier?: string }>("Page.addScriptToEvaluateOnNewDocument", { + source: PICKER_BOOTSTRAP_SOURCE + }).catch(() => null); + const injectionId = registered?.identifier; + try { + await inject(); + const deadline = Date.now() + Math.max(5000, timeoutMs); + while (Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 400)); + const state = await this.evaluate<{ result: { ok: boolean; value?: ElementPickResult; reason?: string } | null; active: boolean }>( + "({ result: window.__rpaPickResult ?? null, active: window.__rpaPickerActive?.active === true })" + ).catch(() => null); + if (!state) continue; // 页面跳转中：等待新文档自动注入 + if (state.result) { + if (state.result.ok && state.result.value) return state.result.value; + throw new RuntimeError("cancelled", state.result.reason || "已取消元素拾取"); } - return trees.join(' >>> '); - }; - const describe = (node) => { - const rect = node.getBoundingClientRect(); - return { - selector: 'css:' + cssPath(node), - tag: node.tagName.toLowerCase(), - id: node.id || undefined, - className: node.getAttribute('class') || undefined, - name: node.getAttribute('name') || undefined, - type: node.getAttribute('type') || undefined, - placeholder: node.getAttribute('placeholder') || undefined, - label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined, - text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined, - value: String(node.value ?? '') || undefined, - url: location.href - }; - }; - const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || ''; - const updateBox = () => { - const node = locked ? selected : target; - if (!node || !box || !badge) return; - const rect = node.getBoundingClientRect(); - box.style.left = Math.max(0, rect.left - 3) + 'px'; - box.style.top = Math.max(0, rect.top - 3) + 'px'; - box.style.width = Math.max(4, rect.width + 6) + 'px'; - box.style.height = Math.max(4, rect.height + 6) + 'px'; - const tag = node.tagName.toLowerCase(); - const name = nodeName(node); - if (locked) { - badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消'; - } else { - badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消'; - } - badge.style.left = Math.max(0, rect.left) + 'px'; - badge.style.top = Math.max(0, rect.top - 34) + 'px'; - }; - const interactiveSelector = [ - 'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option', - '[role]', '[onclick]', '[contenteditable]', '[tabindex]' - ].join(','); - const closestInteractive = (start) => { - let node = start; - while (node && node.nodeType === 1) { - try { if (node.matches(interactiveSelector)) return node; } catch {} - if (node.parentElement) { - node = node.parentElement; - continue; - } - const root = node.getRootNode(); - node = root instanceof ShadowRoot ? root.host : null; - } - return start; - }; - const firstElement = (event) => { - let deepest = null; - for (const item of event.composedPath()) { - if (item instanceof Element) { - deepest = item; - break; - } - } - if (!deepest && event.target instanceof Element) deepest = event.target; - return closestInteractive(deepest); - }; - const buildChain = (node) => { - const result = []; - let current = node; - while (current && current.nodeType === 1) { - result.unshift(current); - const root = current.getRootNode(); - current = root instanceof ShadowRoot ? root.host : null; - } - return result; - }; - const lock = () => { - if (locked || !target) return; - locked = true; - chain = buildChain(target); - chainIndex = Math.max(0, chain.length - 1); - selected = chain[chainIndex] || target; - updateBox(); - }; - const adjustLevel = (delta) => { - if (!locked || !chain.length) return; - const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta)); - if (next === chainIndex) return; - chainIndex = next; - selected = chain[chainIndex]; - updateBox(); - }; - const confirm = () => { - if (!locked || !selected) return; - cleanup(); - resolve(describe(selected)); - }; - const move = (event) => { - if (finished || locked) return; - target = firstElement(event); - updateBox(); - }; - const click = (event) => { - if (finished) return; - // 与桌面拾取对齐：鼠标只用于定位，空格锁定/确认，避免误触发页面按钮。 - event.preventDefault(); - event.stopPropagation(); - event.stopImmediatePropagation(); - }; - const key = (event) => { - if (event.key === 'Escape') { - event.preventDefault(); - event.stopPropagation(); - cleanup(); - reject(new Error('已取消元素拾取')); - return; - } - if (event.code === 'Space' && !event.repeat) { - event.preventDefault(); - event.stopPropagation(); - if (locked) confirm(); - else lock(); - return; - } - if (locked && event.key === 'PageUp' && !event.repeat) { - event.preventDefault(); - event.stopPropagation(); - adjustLevel(-1); - return; - } - if (locked && event.key === 'PageDown' && !event.repeat) { - event.preventDefault(); - event.stopPropagation(); - adjustLevel(1); - } - }; - const cleanup = () => { - if (finished) return; - finished = true; - clearTimeout(timer); - window.removeEventListener('mousemove', move, true); - window.removeEventListener('mousedown', click, true); - window.removeEventListener('click', click, true); - window.removeEventListener('scroll', updateBox, true); - window.removeEventListener('resize', updateBox); - window.removeEventListener('keydown', key, true); - host?.remove(); - delete window.__rpaElementPicker; - }; - - host = document.createElement('div'); - host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;'; - root = host.attachShadow({ mode: 'closed' }); - box = document.createElement('div'); - badge = document.createElement('div'); - box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;'; - badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;'; - root.append(box, badge); - document.documentElement.append(host); - window.addEventListener('mousemove', move, true); - window.addEventListener('mousedown', click, true); - window.addEventListener('click', click, true); - window.addEventListener('scroll', updateBox, true); - window.addEventListener('resize', updateBox); - window.addEventListener('keyup', () => {}, true); - window.addEventListener('keydown', key, true); - window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } }; - timer = setTimeout(() => { - cleanup(); - reject(new Error('拾取超时，请重新点击拾取')); - }, timeout); - }))()`); - return result; + if (!state.active) await inject().catch(() => {}); // 注入未生效时兜底重注入 + } + throw new RuntimeError("timeout", "拾取超时，请重新点击拾取"); + } finally { + await this.evaluate( + "(() => { try { window.__rpaElementPicker?.cancelHard?.(); window.__rpaPickerActive = undefined; delete window.__rpaPickResult; } catch {} })()" + ).catch(() => {}); + if (injectionId) { + await this.cdp("Page.removeScriptToEvaluateOnNewDocument", { identifier: injectionId }).catch(() => {}); + } + } } async close(): Promise { @@ -1429,6 +1226,242 @@ declare global { var __rpaExternalSessions: Map | undefined; } +const PICKER_BOOTSTRAP_SOURCE = `(() => { + if (window.__rpaPickerActive && window.__rpaPickerActive.active) return; + document.getElementById('__rpa_pick_host') && document.getElementById('__rpa_pick_host').remove(); + window.__rpaPickResult = null; + + let host = null; + let root = null; + let box = null; + let badge = null; + let target = null; + let locked = false; + let chain = []; + let chainIndex = 0; + let selected = null; + let finished = false; + + const cssEscape = (value) => { + if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value)); + return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char); + }; + const isUnique = (candidate) => { + try { return document.querySelectorAll(candidate).length === 1; } catch { return false; } + }; + const cssPath = (node) => { + const localPath = (start, root) => { + const stable = (item) => { + if (item.id) { + const candidate = '#' + cssEscape(item.id); + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + const testId = item.getAttribute('data-testid'); + if (testId) { + const candidate = '[data-testid="' + cssEscape(testId) + '"]'; + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + return null; + }; + const parts = []; + let current = start; + while (current && current.nodeType === 1) { + const stableCandidate = stable(current); + if (stableCandidate) { + parts.unshift(stableCandidate); + return parts.join(' > '); + } + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); + if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; + } + parts.unshift(part); + current = parent; + } + return parts.join(' > '); + }; + const trees = []; + let current = node; + while (current && current.nodeType === 1) { + const root = current.getRootNode(); + trees.unshift(localPath(current, root)); + current = root instanceof ShadowRoot ? root.host : null; + } + return trees.join(' >>> '); + }; + const describe = (node) => { + const rect = node.getBoundingClientRect(); + return { + selector: 'css:' + cssPath(node), + tag: node.tagName.toLowerCase(), + id: node.id || undefined, + className: node.getAttribute('class') || undefined, + name: node.getAttribute('name') || undefined, + type: node.getAttribute('type') || undefined, + placeholder: node.getAttribute('placeholder') || undefined, + label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined, + text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined, + value: String(node.value ?? '') || undefined, + url: location.href + }; + }; + const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || ''; + const updateBox = () => { + const node = locked ? selected : target; + if (!node || !box || !badge) return; + const rect = node.getBoundingClientRect(); + box.style.left = Math.max(0, rect.left - 3) + 'px'; + box.style.top = Math.max(0, rect.top - 3) + 'px'; + box.style.width = Math.max(4, rect.width + 6) + 'px'; + box.style.height = Math.max(4, rect.height + 6) + 'px'; + const tag = node.tagName.toLowerCase(); + const name = nodeName(node); + if (locked) { + badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消'; + } else { + badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消 · 可直接翻页后继续拾取'; + } + badge.style.left = Math.max(0, rect.left) + 'px'; + badge.style.top = Math.max(0, rect.top - 34) + 'px'; + }; + const interactiveSelector = [ + 'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option', + '[role]', '[onclick]', '[contenteditable]', '[tabindex]' + ].join(','); + const closestInteractive = (start) => { + let node = start; + while (node && node.nodeType === 1) { + try { if (node.matches(interactiveSelector)) return node; } catch {} + if (node.parentElement) { + node = node.parentElement; + continue; + } + const root = node.getRootNode(); + node = root instanceof ShadowRoot ? root.host : null; + } + return start; + }; + const firstElement = (event) => { + let deepest = null; + for (const item of event.composedPath()) { + if (item instanceof Element) { + deepest = item; + break; + } + } + if (!deepest && event.target instanceof Element) deepest = event.target; + return closestInteractive(deepest); + }; + const buildChain = (node) => { + const result = []; + let current = node; + while (current && current.nodeType === 1) { + result.unshift(current); + const root = current.getRootNode(); + current = root instanceof ShadowRoot ? root.host : null; + } + return result; + }; + const lock = () => { + if (locked || !target) return; + locked = true; + chain = buildChain(target); + chainIndex = Math.max(0, chain.length - 1); + selected = chain[chainIndex] || target; + updateBox(); + }; + const adjustLevel = (delta) => { + if (!locked || !chain.length) return; + const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta)); + if (next === chainIndex) return; + chainIndex = next; + selected = chain[chainIndex]; + updateBox(); + }; + const confirm = () => { + if (!locked || !selected) return; + finish({ ok: true, value: describe(selected) }); + }; + const move = (event) => { + if (finished || locked) return; + target = firstElement(event); + updateBox(); + }; + const click = (event) => { + if (finished) return; + event.preventDefault(); + event.stopPropagation(); + event.stopImmediatePropagation(); + }; + const key = (event) => { + if (event.key === 'Escape') { + event.preventDefault(); + event.stopPropagation(); + finish({ ok: false, reason: '已取消元素拾取' }); + return; + } + if (event.code === 'Space' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + if (locked) confirm(); + else lock(); + return; + } + if (locked && event.key === 'PageUp' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + adjustLevel(-1); + return; + } + if (locked && event.key === 'PageDown' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + adjustLevel(1); + } + }; + const cleanup = () => { + if (finished) return; + finished = true; + window.removeEventListener('mousemove', move, true); + window.removeEventListener('mousedown', click, true); + window.removeEventListener('click', click, true); + window.removeEventListener('scroll', updateBox, true); + window.removeEventListener('resize', updateBox); + window.removeEventListener('keydown', key, true); + host && host.remove(); + window.__rpaElementPicker = null; + window.__rpaPickerActive = { active: false }; + }; + const finish = (payload) => { + cleanup(); + window.__rpaPickResult = payload; + }; + + host = document.createElement('div'); + host.id = '__rpa_pick_host'; + host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;'; + root = host.attachShadow({ mode: 'closed' }); + box = document.createElement('div'); + badge = document.createElement('div'); + box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;'; + badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;'; + root.append(box, badge); + document.documentElement.append(host); + window.addEventListener('mousemove', move, true); + window.addEventListener('mousedown', click, true); + window.addEventListener('click', click, true); + window.addEventListener('scroll', updateBox, true); + window.addEventListener('resize', updateBox); + window.addEventListener('keydown', key, true); + window.__rpaElementPicker = { + cancel: () => { if (!finished) finish({ ok: false, reason: '已取消元素拾取' }); }, + cancelHard: () => { cleanup(); } + }; + window.__rpaPickerActive = { active: true }; +})();`; + function externalSessionCache(): Map { if (!globalThis.__rpaExternalSessions) globalThis.__rpaExternalSessions = new Map(); return globalThis.__rpaExternalSessions;