feat: 新增本地UI模型AI拾取与识别实验台

src/main/browser/session.ts

@@ -233,6 +233,18 @@ export class BrowserSession implements BrowserLike {
             return !name || accessibleName.includes(name);
           }) || null;
         }
+        if (spec.kind === 'css') {
+          const segments = String(spec.value).split('>>>').map((item) => item.trim()).filter(Boolean);
+          let root = document;
+          for (let index = 0; index < segments.length; index += 1) {
+            const found = root.querySelector(segments[index]);
+            if (!found) return null;
+            if (index === segments.length - 1) return found;
+            root = found.shadowRoot;
+            if (!root) return null;
+          }
+          return null;
+        }
         return document.querySelector(spec.value);
       };
       const el = find();
@@ -269,33 +281,46 @@ export class BrowserSession implements BrowserLike {
         try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
       };
       const cssPath = (node) => {
-        if (node.id) {
-          const candidate = '#' + cssEscape(node.id);
-          if (isUnique(candidate)) return candidate;
-        }
-        const testId = node.getAttribute('data-testid');
-        if (testId) {
-          const candidate = '[data-testid="' + cssEscape(testId) + '"]';
-          if (isUnique(candidate)) return candidate;
-        }
-        const parts = [];
+        const localPath = (start, root) => {
+          const stable = (item) => {
+            if (item.id) {
+              const candidate = '#' + cssEscape(item.id);
+              try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
+            }
+            const testId = item.getAttribute('data-testid');
+            if (testId) {
+              const candidate = '[data-testid="' + cssEscape(testId) + '"]';
+              try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
+            }
+            return null;
+          };
+          const parts = [];
+          let current = start;
+          while (current && current.nodeType === 1) {
+            const stableCandidate = stable(current);
+            if (stableCandidate) {
+              parts.unshift(stableCandidate);
+              return parts.join(' > ');
+            }
+            let part = current.tagName.toLowerCase();
+            const parent = current.parentElement;
+            if (parent) {
+              const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
+              if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
+            }
+            parts.unshift(part);
+            current = parent;
+          }
+          return parts.join(' > ');
+        };
+        const trees = [];
         let current = node;
         while (current && current.nodeType === 1) {
-          let part = current.tagName.toLowerCase();
-          if (current.id) {
-            part += '#' + cssEscape(current.id);
-            parts.unshift(part);
-            break;
-          }
-          const parent = current.parentElement;
-          if (parent) {
-            const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
-            if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
-          }
-          parts.unshift(part);
-          current = parent;
+          const root = current.getRootNode();
+          trees.unshift(localPath(current, root));
+          current = root instanceof ShadowRoot ? root.host : null;
         }
-        return parts.join(' > ');
+        return trees.join(' >>> ');
       };
       const describe = (node) => ({
         kind: 'css',
@@ -1143,33 +1168,46 @@ export class BrowserSession implements BrowserLike {
         try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
       };
       const cssPath = (node) => {
-        if (node.id) {
-          const candidate = '#' + cssEscape(node.id);
-          if (isUnique(candidate)) return candidate;
-        }
-        const testId = node.getAttribute('data-testid');
-        if (testId) {
-          const candidate = '[data-testid="' + cssEscape(testId) + '"]';
-          if (isUnique(candidate)) return candidate;
-        }
-        const parts = [];
+        const localPath = (start, root) => {
+          const stable = (item) => {
+            if (item.id) {
+              const candidate = '#' + cssEscape(item.id);
+              try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
+            }
+            const testId = item.getAttribute('data-testid');
+            if (testId) {
+              const candidate = '[data-testid="' + cssEscape(testId) + '"]';
+              try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
+            }
+            return null;
+          };
+          const parts = [];
+          let current = start;
+          while (current && current.nodeType === 1) {
+            const stableCandidate = stable(current);
+            if (stableCandidate) {
+              parts.unshift(stableCandidate);
+              return parts.join(' > ');
+            }
+            let part = current.tagName.toLowerCase();
+            const parent = current.parentElement;
+            if (parent) {
+              const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
+              if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
+            }
+            parts.unshift(part);
+            current = parent;
+          }
+          return parts.join(' > ');
+        };
+        const trees = [];
         let current = node;
         while (current && current.nodeType === 1) {
-          let part = current.tagName.toLowerCase();
-          if (current.id) {
-            part += '#' + cssEscape(current.id);
-            parts.unshift(part);
-            break;
-          }
-          const parent = current.parentElement;
-          if (parent) {
-            const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
-            if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
-          }
-          parts.unshift(part);
-          current = parent;
+          const root = current.getRootNode();
+          trees.unshift(localPath(current, root));
+          current = root instanceof ShadowRoot ? root.host : null;
         }
-        return parts.join(' > ');
+        return trees.join(' >>> ');
       };
       const describe = (node) => {
         const rect = node.getBoundingClientRect();
@@ -1200,9 +1238,33 @@ export class BrowserSession implements BrowserLike {
         badge.style.left = Math.max(0, rect.left) + 'px';
         badge.style.top = Math.max(0, rect.top - 30) + 'px';
       };
+      const interactiveSelector = [
+        'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
+        '[role]', '[onclick]', '[contenteditable]', '[tabindex]'
+      ].join(',');
+      const closestInteractive = (start) => {
+        let node = start;
+        while (node && node.nodeType === 1) {
+          try { if (node.matches(interactiveSelector)) return node; } catch {}
+          if (node.parentElement) {
+            node = node.parentElement;
+            continue;
+          }
+          const root = node.getRootNode();
+          node = root instanceof ShadowRoot ? root.host : null;
+        }
+        return start;
+      };
       const firstElement = (event) => {
-        for (const item of event.composedPath()) if (item instanceof Element) return item;
-        return event.target instanceof Element ? event.target : null;
+        let deepest = null;
+        for (const item of event.composedPath()) {
+          if (item instanceof Element) {
+            deepest = item;
+            break;
+          }
+        }
+        if (!deepest && event.target instanceof Element) deepest = event.target;
+        return closestInteractive(deepest);
       };
       const move = (event) => {
         if (finished) return;