feat: align browser picker with desktop interaction

This commit is contained in:
yu 2026-11-02 14:12:41 +08:00
parent 9b9c6f17e4
commit 1c85a90f1f
2 changed files with 71 additions and 13 deletions

View File

@ -1167,6 +1167,10 @@ export class BrowserSession implements BrowserLike {
let box = null;
let badge = null;
let target = null;
let locked = false;
let chain = [];
let chainIndex = 0;
let selected = null;
let timer = 0;
let finished = false;
@ -1235,18 +1239,24 @@ export class BrowserSession implements BrowserLike {
url: location.href
};
};
const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || '';
const updateBox = () => {
if (!target || !box || !badge) return;
const rect = target.getBoundingClientRect();
const node = locked ? selected : target;
if (!node || !box || !badge) return;
const rect = node.getBoundingClientRect();
box.style.left = Math.max(0, rect.left - 3) + 'px';
box.style.top = Math.max(0, rect.top - 3) + 'px';
box.style.width = Math.max(4, rect.width + 6) + 'px';
box.style.height = Math.max(4, rect.height + 6) + 'px';
const tag = target.tagName.toLowerCase();
const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || '';
badge.textContent = '拾取:' + tag + (name ? ' · ' + name.slice(0, 60) : '');
const tag = node.tagName.toLowerCase();
const name = nodeName(node);
if (locked) {
badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消';
} else {
badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消';
}
badge.style.left = Math.max(0, rect.left) + 'px';
badge.style.top = Math.max(0, rect.top - 30) + 'px';
badge.style.top = Math.max(0, rect.top - 34) + 'px';
};
const interactiveSelector = [
'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
@ -1276,20 +1286,48 @@ export class BrowserSession implements BrowserLike {
if (!deepest && event.target instanceof Element) deepest = event.target;
return closestInteractive(deepest);
};
const buildChain = (node) => {
const result = [];
let current = node;
while (current && current.nodeType === 1) {
result.unshift(current);
const root = current.getRootNode();
current = root instanceof ShadowRoot ? root.host : null;
}
return result;
};
const lock = () => {
if (locked || !target) return;
locked = true;
chain = buildChain(target);
chainIndex = Math.max(0, chain.length - 1);
selected = chain[chainIndex] || target;
updateBox();
};
const adjustLevel = (delta) => {
if (!locked || !chain.length) return;
const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta));
if (next === chainIndex) return;
chainIndex = next;
selected = chain[chainIndex];
updateBox();
};
const confirm = () => {
if (!locked || !selected) return;
cleanup();
resolve(describe(selected));
};
const move = (event) => {
if (finished) return;
if (finished || locked) return;
target = firstElement(event);
updateBox();
};
const click = (event) => {
if (finished) return;
const selected = firstElement(event);
if (!selected) return;
// 与桌面拾取对齐:鼠标只用于定位,空格锁定/确认,避免误触发页面按钮。
event.preventDefault();
event.stopPropagation();
event.stopImmediatePropagation();
cleanup();
resolve(describe(selected));
};
const key = (event) => {
if (event.key === 'Escape') {
@ -1297,6 +1335,25 @@ export class BrowserSession implements BrowserLike {
event.stopPropagation();
cleanup();
reject(new Error('已取消元素拾取'));
return;
}
if (event.code === 'Space' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
if (locked) confirm();
else lock();
return;
}
if (locked && event.key === 'PageUp' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(-1);
return;
}
if (locked && event.key === 'PageDown' && !event.repeat) {
event.preventDefault();
event.stopPropagation();
adjustLevel(1);
}
};
const cleanup = () => {
@ -1327,6 +1384,7 @@ export class BrowserSession implements BrowserLike {
window.addEventListener('click', click, true);
window.addEventListener('scroll', updateBox, true);
window.addEventListener('resize', updateBox);
window.addEventListener('keyup', () => {}, true);
window.addEventListener('keydown', key, true);
window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
timer = setTimeout(() => {

View File

@ -664,8 +664,8 @@ function FieldControl({
</button>
</>
) : (
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
{picking ? "拾取中" : "拾取"}
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从浏览器拾取元素(移到目标按空格锁定,PageUp/PageDown 调层级)">
{picking ? "浏览器拾取中" : "浏览器拾取"}
</button>
)}
</div>