元素拾取支持页面跳转后继续拾取; 恢复 ws 根依赖声明修复启动崩溃

- pickElement 重构: 拾取脚本经 Page.addScriptToEvaluateOnNewDocument 随新文档自动重注入,
  主进程轮询 __rpaPickResult, 用户拾取期间可翻页/搜索/过滑块, 页面跳转不再使拾取失效
- 恢复 package.json ws 根依赖与 electron.vite.config include:[ws]
  (合并时被覆盖导致 dev 启动报 bufferutil 解析错误)
This commit is contained in:
yu 2026-11-05 10:07:48 +08:00
parent 2ae1010f67
commit 5250bf1e35
3 changed files with 270 additions and 236 deletions

1
package-lock.json generated
View File

@ -34,6 +34,7 @@
"react-dom": "^18.3.1",
"sm-crypto": "^0.5.8",
"tesseract.js": "^7.0.0",
"ws": "^8.18.0",
"zustand": "^5.0.2"
},
"devDependencies": {

View File

@ -46,7 +46,7 @@
"react-dom": "^18.3.1",
"sm-crypto": "^0.5.8",
"tesseract.js": "^7.0.0",
"ws": "^8.22.0",
"ws": "^8.18.0",
"zustand": "^5.0.2"
},
"devDependencies": {

View File

@ -1153,7 +1153,8 @@ export class BrowserSession implements BrowserLike {
}
async pickElement(timeoutMs = 90000): Promise<import("../../shared/types").ElementPickResult> {
// 拾取脚本经 Runtime.evaluate 注入并等待用户点击,内置/外部浏览器行为一致
// 拾取脚本注入当前文档,并经 Page.addScriptToEvaluateOnNewDocument 随页面跳转自动重注入;
// 主进程轮询 window.__rpaPickResult——用户在拾取期间可随意翻页/搜索/过滑块,停在哪就能拾哪。
const url = await this.currentUrl();
if (!url || url === "about:blank") {
throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」,再拾取元素" });
@ -1162,241 +1163,37 @@ export class BrowserSession implements BrowserLike {
this.win.show();
this.win.focus();
}
const result = await this.evaluate<ElementPickResult>(`(() => new Promise((resolve, reject) => {
const timeout = ${JSON.stringify(timeoutMs)};
if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel();
let host = null;
let root = null;
let box = null;
let badge = null;
let target = null;
let locked = false;
let chain = [];
let chainIndex = 0;
let selected = null;
let timer = 0;
let finished = false;
const cssEscape = (value) => {
if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value));
return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char);
};
const isUnique = (candidate) => {
try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
};
const cssPath = (node) => {
const localPath = (start, root) => {
const stable = (item) => {
if (item.id) {
const candidate = '#' + cssEscape(item.id);
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
}
const testId = item.getAttribute('data-testid');
if (testId) {
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
}
return null;
};
const parts = [];
let current = start;
while (current && current.nodeType === 1) {
const stableCandidate = stable(current);
if (stableCandidate) {
parts.unshift(stableCandidate);
return parts.join(' > ');
}
let part = current.tagName.toLowerCase();
const parent = current.parentElement;
if (parent) {
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
}
parts.unshift(part);
current = parent;
}
return parts.join(' > ');
};
const trees = [];
let current = node;
while (current && current.nodeType === 1) {
const root = current.getRootNode();
trees.unshift(localPath(current, root));
current = root instanceof ShadowRoot ? root.host : null;
const inject = () =>
this.evaluate(`(() => { try { delete window.__rpaPickResult; ${PICKER_BOOTSTRAP_SOURCE} } catch {} })()`);
await this.cdp("Page.enable").catch(() => {});
const registered = await this.cdp<{ identifier?: string }>("Page.addScriptToEvaluateOnNewDocument", {
source: PICKER_BOOTSTRAP_SOURCE
}).catch(() => null);
const injectionId = registered?.identifier;
try {
await inject();
const deadline = Date.now() + Math.max(5000, timeoutMs);
while (Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 400));
const state = await this.evaluate<{ result: { ok: boolean; value?: ElementPickResult; reason?: string } | null; active: boolean }>(
"({ result: window.__rpaPickResult ?? null, active: window.__rpaPickerActive?.active === true })"
).catch(() => null);
if (!state) continue; // 页面跳转中:等待新文档自动注入
if (state.result) {
if (state.result.ok && state.result.value) return state.result.value;
throw new RuntimeError("cancelled", state.result.reason || "已取消元素拾取");
}
return trees.join(' >>> ');
};
const describe = (node) => {
const rect = node.getBoundingClientRect();
return {
selector: 'css:' + cssPath(node),
tag: node.tagName.toLowerCase(),
id: node.id || undefined,
className: node.getAttribute('class') || undefined,
name: node.getAttribute('name') || undefined,
type: node.getAttribute('type') || undefined,
placeholder: node.getAttribute('placeholder') || undefined,
label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined,
text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined,
value: String(node.value ?? '') || undefined,
url: location.href
};
};
const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || '';
const updateBox = () => {
const node = locked ? selected : target;
if (!node || !box || !badge) return;
const rect = node.getBoundingClientRect();
box.style.left = Math.max(0, rect.left - 3) + 'px';
box.style.top = Math.max(0, rect.top - 3) + 'px';
box.style.width = Math.max(4, rect.width + 6) + 'px';
box.style.height = Math.max(4, rect.height + 6) + 'px';
const tag = node.tagName.toLowerCase();
const name = nodeName(node);
if (locked) {
badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消';
} else {
badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消';
}
badge.style.left = Math.max(0, rect.left) + 'px';
badge.style.top = Math.max(0, rect.top - 34) + 'px';
};
const interactiveSelector = [
'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
'[role]', '[onclick]', '[contenteditable]', '[tabindex]'
].join(',');
const closestInteractive = (start) => {
let node = start;
while (node && node.nodeType === 1) {
try { if (node.matches(interactiveSelector)) return node; } catch {}
if (node.parentElement) {
node = node.parentElement;
continue;
}
const root = node.getRootNode();
node = root instanceof ShadowRoot ? root.host : null;
}
return start;
};
const firstElement = (event) => {
let deepest = null;
for (const item of event.composedPath()) {
if (item instanceof Element) {
deepest = item;
break;
}
}
if (!deepest && event.target instanceof Element) deepest = event.target;
return closestInteractive(deepest);
};
const buildChain = (node) => {
const result = [];
let current = node;
while (current && current.nodeType === 1) {
result.unshift(current);
const root = current.getRootNode();
current = root instanceof ShadowRoot ? root.host : null;
}
return result;
};
const lock = () => {
if (locked || !target) return;
locked = true;
chain = buildChain(target);
chainIndex = Math.max(0, chain.length - 1);
selected = chain[chainIndex] || target;
updateBox();
};
const adjustLevel = (delta) => {
if (!locked || !chain.length) return;
const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta));
if (next === chainIndex) return;
chainIndex = next;
selected = chain[chainIndex];
updateBox();
};
const confirm = () => {
if (!locked || !selected) return;
cleanup();
resolve(describe(selected));
};
const move = (event) => {
if (finished || locked) return;
target = firstElement(event);
updateBox();
};
const click = (event) => {
if (finished) return;
// 与桌面拾取对齐:鼠标只用于定位,空格锁定/确认,避免误触发页面按钮。
event.preventDefault();
event.stopPropagation();
event.stopImmediatePropagation();
};
const key = (event) => {
if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
cleanup();
reject(new Error('已取消元素拾取'));
return;
}
if (event.code === 'Space' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
if (locked) confirm();
else lock();
return;
}
if (locked && event.key === 'PageUp' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(-1);
return;
}
if (locked && event.key === 'PageDown' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(1);
}
};
const cleanup = () => {
if (finished) return;
finished = true;
clearTimeout(timer);
window.removeEventListener('mousemove', move, true);
window.removeEventListener('mousedown', click, true);
window.removeEventListener('click', click, true);
window.removeEventListener('scroll', updateBox, true);
window.removeEventListener('resize', updateBox);
window.removeEventListener('keydown', key, true);
host?.remove();
delete window.__rpaElementPicker;
};
host = document.createElement('div');
host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;';
root = host.attachShadow({ mode: 'closed' });
box = document.createElement('div');
badge = document.createElement('div');
box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;';
badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;';
root.append(box, badge);
document.documentElement.append(host);
window.addEventListener('mousemove', move, true);
window.addEventListener('mousedown', click, true);
window.addEventListener('click', click, true);
window.addEventListener('scroll', updateBox, true);
window.addEventListener('resize', updateBox);
window.addEventListener('keyup', () => {}, true);
window.addEventListener('keydown', key, true);
window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
timer = setTimeout(() => {
cleanup();
reject(new Error('拾取超时,请重新点击拾取'));
}, timeout);
}))()`);
return result;
if (!state.active) await inject().catch(() => {}); // 注入未生效时兜底重注入
}
throw new RuntimeError("timeout", "拾取超时,请重新点击拾取");
} finally {
await this.evaluate(
"(() => { try { window.__rpaElementPicker?.cancelHard?.(); window.__rpaPickerActive = undefined; delete window.__rpaPickResult; } catch {} })()"
).catch(() => {});
if (injectionId) {
await this.cdp("Page.removeScriptToEvaluateOnNewDocument", { identifier: injectionId }).catch(() => {});
}
}
}
async close(): Promise<void> {
@ -1429,6 +1226,242 @@ declare global {
var __rpaExternalSessions: Map<string, BrowserSession> | undefined;
}
const PICKER_BOOTSTRAP_SOURCE = `(() => {
if (window.__rpaPickerActive && window.__rpaPickerActive.active) return;
document.getElementById('__rpa_pick_host') && document.getElementById('__rpa_pick_host').remove();
window.__rpaPickResult = null;
let host = null;
let root = null;
let box = null;
let badge = null;
let target = null;
let locked = false;
let chain = [];
let chainIndex = 0;
let selected = null;
let finished = false;
const cssEscape = (value) => {
if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value));
return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char);
};
const isUnique = (candidate) => {
try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
};
const cssPath = (node) => {
const localPath = (start, root) => {
const stable = (item) => {
if (item.id) {
const candidate = '#' + cssEscape(item.id);
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
}
const testId = item.getAttribute('data-testid');
if (testId) {
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
}
return null;
};
const parts = [];
let current = start;
while (current && current.nodeType === 1) {
const stableCandidate = stable(current);
if (stableCandidate) {
parts.unshift(stableCandidate);
return parts.join(' > ');
}
let part = current.tagName.toLowerCase();
const parent = current.parentElement;
if (parent) {
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
}
parts.unshift(part);
current = parent;
}
return parts.join(' > ');
};
const trees = [];
let current = node;
while (current && current.nodeType === 1) {
const root = current.getRootNode();
trees.unshift(localPath(current, root));
current = root instanceof ShadowRoot ? root.host : null;
}
return trees.join(' >>> ');
};
const describe = (node) => {
const rect = node.getBoundingClientRect();
return {
selector: 'css:' + cssPath(node),
tag: node.tagName.toLowerCase(),
id: node.id || undefined,
className: node.getAttribute('class') || undefined,
name: node.getAttribute('name') || undefined,
type: node.getAttribute('type') || undefined,
placeholder: node.getAttribute('placeholder') || undefined,
label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined,
text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined,
value: String(node.value ?? '') || undefined,
url: location.href
};
};
const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || '';
const updateBox = () => {
const node = locked ? selected : target;
if (!node || !box || !badge) return;
const rect = node.getBoundingClientRect();
box.style.left = Math.max(0, rect.left - 3) + 'px';
box.style.top = Math.max(0, rect.top - 3) + 'px';
box.style.width = Math.max(4, rect.width + 6) + 'px';
box.style.height = Math.max(4, rect.height + 6) + 'px';
const tag = node.tagName.toLowerCase();
const name = nodeName(node);
if (locked) {
badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消';
} else {
badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消 · 可直接翻页后继续拾取';
}
badge.style.left = Math.max(0, rect.left) + 'px';
badge.style.top = Math.max(0, rect.top - 34) + 'px';
};
const interactiveSelector = [
'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
'[role]', '[onclick]', '[contenteditable]', '[tabindex]'
].join(',');
const closestInteractive = (start) => {
let node = start;
while (node && node.nodeType === 1) {
try { if (node.matches(interactiveSelector)) return node; } catch {}
if (node.parentElement) {
node = node.parentElement;
continue;
}
const root = node.getRootNode();
node = root instanceof ShadowRoot ? root.host : null;
}
return start;
};
const firstElement = (event) => {
let deepest = null;
for (const item of event.composedPath()) {
if (item instanceof Element) {
deepest = item;
break;
}
}
if (!deepest && event.target instanceof Element) deepest = event.target;
return closestInteractive(deepest);
};
const buildChain = (node) => {
const result = [];
let current = node;
while (current && current.nodeType === 1) {
result.unshift(current);
const root = current.getRootNode();
current = root instanceof ShadowRoot ? root.host : null;
}
return result;
};
const lock = () => {
if (locked || !target) return;
locked = true;
chain = buildChain(target);
chainIndex = Math.max(0, chain.length - 1);
selected = chain[chainIndex] || target;
updateBox();
};
const adjustLevel = (delta) => {
if (!locked || !chain.length) return;
const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta));
if (next === chainIndex) return;
chainIndex = next;
selected = chain[chainIndex];
updateBox();
};
const confirm = () => {
if (!locked || !selected) return;
finish({ ok: true, value: describe(selected) });
};
const move = (event) => {
if (finished || locked) return;
target = firstElement(event);
updateBox();
};
const click = (event) => {
if (finished) return;
event.preventDefault();
event.stopPropagation();
event.stopImmediatePropagation();
};
const key = (event) => {
if (event.key === 'Escape') {
event.preventDefault();
event.stopPropagation();
finish({ ok: false, reason: '已取消元素拾取' });
return;
}
if (event.code === 'Space' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
if (locked) confirm();
else lock();
return;
}
if (locked && event.key === 'PageUp' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(-1);
return;
}
if (locked && event.key === 'PageDown' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(1);
}
};
const cleanup = () => {
if (finished) return;
finished = true;
window.removeEventListener('mousemove', move, true);
window.removeEventListener('mousedown', click, true);
window.removeEventListener('click', click, true);
window.removeEventListener('scroll', updateBox, true);
window.removeEventListener('resize', updateBox);
window.removeEventListener('keydown', key, true);
host && host.remove();
window.__rpaElementPicker = null;
window.__rpaPickerActive = { active: false };
};
const finish = (payload) => {
cleanup();
window.__rpaPickResult = payload;
};
host = document.createElement('div');
host.id = '__rpa_pick_host';
host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;';
root = host.attachShadow({ mode: 'closed' });
box = document.createElement('div');
badge = document.createElement('div');
box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;';
badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;';
root.append(box, badge);
document.documentElement.append(host);
window.addEventListener('mousemove', move, true);
window.addEventListener('mousedown', click, true);
window.addEventListener('click', click, true);
window.addEventListener('scroll', updateBox, true);
window.addEventListener('resize', updateBox);
window.addEventListener('keydown', key, true);
window.__rpaElementPicker = {
cancel: () => { if (!finished) finish({ ok: false, reason: '已取消元素拾取' }); },
cancelHard: () => { cleanup(); }
};
window.__rpaPickerActive = { active: true };
})();`;
function externalSessionCache(): Map<string, BrowserSession> {
if (!globalThis.__rpaExternalSessions) globalThis.__rpaExternalSessions = new Map();
return globalThis.__rpaExternalSessions;