release: v0.8.0 — browser extension for Chrome/Edge/Firefox
Some checks failed
Stable Release / stable (push) Failing after 48s

Assisted-by: GLM-5.1 via Crush <crush@charm.land>
This commit is contained in:
Augustin
2026-04-27 16:51:00 +02:00
29 changed files with 6211 additions and 48 deletions

View File

@@ -32,13 +32,21 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Cache Node modules
- name: Cache Node modules (web)
uses: actions/cache@v4
with:
path: web/node_modules
key: ${{ runner.os }}-node-${{ hashFiles('web/package-lock.json') }}
key: ${{ runner.os }}-node-web-${{ hashFiles('web/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
${{ runner.os }}-node-web-
- name: Cache Node modules (extension)
uses: actions/cache@v4
with:
path: extension/node_modules
key: ${{ runner.os }}-node-ext-${{ hashFiles('extension/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-ext-
- name: Download Go dependencies
run: go mod download
@@ -49,6 +57,14 @@ jobs:
npm ci
npm run build
- name: Build extension
run: |
cd extension
npm ci
npx wxt zip
npx wxt zip --browser firefox
mv .output/muyue-extension-*.zip ../dist/
- name: Vet
run: go vet ./...
@@ -152,7 +168,7 @@ jobs:
fi
echo "Release ID: ${RELEASE_ID}"
UPLOAD_URL="${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/${RELEASE_ID}/assets"
for file in dist/*.tar.gz dist/*.zip dist/checksums.txt; do
for file in dist/*.tar.gz dist/*.zip dist/checksums.txt dist/muyue-extension-*.zip; do
filename=$(basename "$file")
echo "Uploading ${filename}..."
curl -s -X POST "${UPLOAD_URL}" \

View File

@@ -32,13 +32,21 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Cache Node modules
- name: Cache Node modules (web)
uses: actions/cache@v4
with:
path: web/node_modules
key: ${{ runner.os }}-node-${{ hashFiles('web/package-lock.json') }}
key: ${{ runner.os }}-node-web-${{ hashFiles('web/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
${{ runner.os }}-node-web-
- name: Cache Node modules (extension)
uses: actions/cache@v4
with:
path: extension/node_modules
key: ${{ runner.os }}-node-ext-${{ hashFiles('extension/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-ext-
- name: Download dependencies
run: go mod download
@@ -49,6 +57,14 @@ jobs:
npm ci
npm run build
- name: Build extension
run: |
cd extension
npm ci
npx wxt zip
npx wxt zip --browser firefox
mv .output/muyue-extension-*.zip ../dist/
- name: Vet
run: go vet ./...
@@ -241,7 +257,7 @@ jobs:
fi
echo "Release ID: ${RELEASE_ID}"
UPLOAD_URL="${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases/${RELEASE_ID}/assets"
for file in dist/*.tar.gz dist/*.zip dist/checksums.txt; do
for file in dist/*.tar.gz dist/*.zip dist/checksums.txt dist/muyue-extension-*.zip; do
filename=$(basename "$file")
echo "Uploading ${filename}..."
UPLOAD_RESP=$(curl -s -w "\n%{http_code}" -X POST "${UPLOAD_URL}" \

View File

@@ -30,13 +30,21 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Cache Node modules
- name: Cache Node modules (web)
uses: actions/cache@v4
with:
path: web/node_modules
key: ${{ runner.os }}-node-${{ hashFiles('web/package-lock.json') }}
key: ${{ runner.os }}-node-web-${{ hashFiles('web/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
${{ runner.os }}-node-web-
- name: Cache Node modules (extension)
uses: actions/cache@v4
with:
path: extension/node_modules
key: ${{ runner.os }}-node-ext-${{ hashFiles('extension/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-ext-
- name: Download dependencies
run: go mod download
@@ -47,13 +55,20 @@ jobs:
npm ci
npm run build
- name: Build extension
run: |
cd extension
npm ci
npm run build
npm run build:firefox
- name: Vet
run: go vet ./...
- name: Test
run: go test ./... -v -race -timeout 60s
- name: Build
- name: Build binary
run: |
go build -o muyue ./cmd/muyue/
./muyue version

5
.gitignore vendored
View File

@@ -32,3 +32,8 @@ vendor/
# Frontend (web/.gitignore handles specifics)
web/node_modules/
# Extension build artifacts
extension/node_modules/
extension/.output/
extension/.wxt/

View File

@@ -4,6 +4,99 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## v0.8.0
### Extension navigateur — Chrome, Edge, Firefox
Nouvelle fonctionnalité majeure : une extension navigateur multi-plateforme qui remplace l'injection manuelle du snippet JS pour les tests pilotés par l'IA.
**Ce que fait l'extension :**
- **Auto-injection** du client Muyue sur chaque page HTTP/HTTPS — plus besoin de copier-coller le snippet dans la console DevTools
- **Capture console** temps réel (log/warn/error/debug + window.onerror + unhandledrejection) transmise au serveur Muyue
- **Screenshots natifs** via `chrome.tabs.captureVisibleTab` / `browser.tabs.captureVisibleTab` — pixels parfaits, pas le hack SVG foreignObject
- **Side Panel** (Chrome/Edge) et **Sidebar** (Firefox) pour monitoring statut serveur + sessions connectées
- **Popup toolbar** : statut serveur, nombre de sessions, erreurs console, lien vers le dashboard
- **Badge dynamique** : nombre de sessions connectées (vert) ou statut serveur (rouge/orange)
- **Détection URL** via interception History API (pushState, replaceState, popstate) + MutationObserver + fallback polling — survit aux navigations SPA
- **Auto-reconnect** avec backoff exponentiel en cas de déconnexion transitoire
- **Compatible Firefox** via Manifest V2 (sidebar_action) et Chrome/Edge via Manifest V3 (sidePanel API)
**Architecture technique :**
- **WXT framework** (build multi-navigateur) avec Vite 8
- **Content script** : même protocole WS que le snippet existant — aucun changement backend nécessaire
- **Background service worker** : `chrome.alarms` pour health checks périodiques (pas de `setInterval`), `chrome.storage.local` pour la config (pas de `localStorage` en MV3)
- **Builds** : `npm run build` (Chrome MV3) + `npm run build:firefox` (Firefox MV2) + `npm run zip` (packages stores)
- **CI** : les 3 workflows (PR, beta, stable) buildent l'extension et attachent les `.zip` aux releases
**Fichiers ajoutés :**
```
extension/
├── package.json, wxt.config.js, .gitignore, README.md
├── public/icon/ # Icons copiés depuis assets/
└── src/
├── entrypoints/
│ ├── background.js # Service worker (token, badge, screenshots)
│ ├── content.js # Auto-injection WS + console capture + History API
│ ├── popup/ # HTML + JS du popup toolbar
│ └── sidepanel/ # HTML + JS du side panel / sidebar
├── lib/
│ ├── config.js # Storage async (chrome.storage + localStorage)
│ └── page-rpc.js # DOM RPC (list_clickables, click, type, eval)
└── styles/panel.css # Thème cyberpunk cohérent avec Muyue
```
**Autres changements :**
- **CI** : les 3 workflows (`ci-pr.yml`, `ci-develop.yml`, `ci-main.yml`) buildent l'extension et attachent les `.zip` aux releases
- **Makefile** : cibles `ext`, `ext-chrome`, `ext-firefox`, `ext-zip` ajoutées
- **README** : section "Browser Extension" ajoutée avec instructions install + dev
- **Version** : bump 0.7.9 → 0.8.0
## v0.7.9
### Tests pilotés par l'IA — robustesse + captures d'écran
Quatre problèmes signalés par l'utilisateur :
1. **Connexion perdue à chaque reload / navigation**. Le token était à usage unique (5 min TTL) et tombait dès la première reconnexion → l'IA perdait totalement la session.
2. **Page nouvellement ouverte invisible à l'IA**. Conséquence du même bug ci-dessus + JS context détruit à la navigation.
3. **Pas de captures d'écran**. L'IA ne pouvait pas prouver visuellement l'état d'une page.
4. **L'IA se perd en boucle d'outils** : ~150 appels pour l'équivalent de 5 actions humaines, parce qu'elle re-listait les éléments cliquables après chaque clic.
**Fixes** :
- **Token réutilisable avec TTL coulissant** (`ConsumeToken` ne supprime plus le token, refresh la TTL à 60 min sur chaque utilisation). L'utilisateur peut re-coller le même snippet de l'onglet Tests sans avoir à régénérer un token, et la session reprend transparente.
- **Auto-reconnect dans le snippet** avec backoff exponentiel (500ms × tentative, max 5 tentatives = ~2,5s). Couvre les déconnexions transitoires (réseau, hibernation, redémarrage du serveur Muyue). Pour une vraie navigation full-page (URL change, JS context détruit), aucun JS ne peut survivre — l'utilisateur doit recoller le snippet, mais c'est immédiat car le token reste valide.
- **Nouvelle action `screenshot`** : le snippet capture la viewport (ou un sélecteur via `selector`) en SVG `foreignObject` + canvas, retourne un data URL base64. Le serveur décode et sauve dans `~/.muyue/screenshots/<filename>.png` (nom personnalisable via `filename`, sinon timestamp). Best-effort — CSS externes / images cross-origin / iframes peuvent ne pas apparaître ; les sélecteurs sont sanitisés (pas d'évasion vers d'autres dossiers).
- **Stratégie de test re-écrite dans le system prompt Studio** : règle d'or *"ne PAS ré-appeler `list_clickables` après chaque clic"*. Tableau des actions avec leur coût relatif (`summary` cher mais utile au début, `eval` ciblé > `list_clickables` complet, etc.). Format de rapport final standardisé (✓ / ✗ / ⚠ / 📸).
Inclut également **v0.7.8** (fix régression v0.7.6 : `unsafe.Pointer(uintptr(hPC))` au lieu de `&hPC` dans `UpdateProcThreadAttribute(PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE)` — corrige les terminaux qui s'ouvraient en fenêtre externe).
## v0.7.8
### Fix régression v0.7.6 : terminaux ouverts en fenêtre externe
Symptôme rapporté : depuis v0.7.6, cliquer sur PowerShell / cmd dans l'onglet Terminal ouvre une **fenêtre console séparée** au lieu de s'afficher dans le tab xterm.js (régression — v0.7.5 fonctionnait).
**Cause** : le binding ConPTY introduit en v0.7.6 passait `&hPC` (pointeur vers la variable Go locale) à `UpdateProcThreadAttribute(PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE, …)`. Or cet attribut est un quirk de l'API Win32 : `lpValue` doit être la **valeur du handle** (cast en `PVOID`), **pas** un pointeur vers la variable. Avec `&hPC`, le kernel lisait des octets aléatoires, l'attribut PSEUDOCONSOLE était silencieusement ignoré, et `CreateProcessW` créait une nouvelle console pour l'enfant — d'où la fenêtre externe.
**Fix** (1 ligne) :
```go
// Avant
unsafe.Pointer(&hPC)
// Après
unsafe.Pointer(uintptr(hPC)) // le HPCON value comme PVOID
```
Référence : Microsoft EchoCon sample + bibliothèques Go ConPTY existantes (`UserExistsError/conpty`, `aymanbagabas/go-pty`) utilisent toutes la valeur du handle directement.
Conséquence : terminaux PowerShell / cmd / WSL s'ouvrent à nouveau **dans** le tab xterm.js avec TTY complet (ANSI, prompt couleur, vim, etc.).
## v0.7.7
### Changes since v0.7.6

View File

@@ -7,7 +7,9 @@ NODE ?= node
NPM ?= npm
WEB_DIR = web
.PHONY: build install clean test test-short run scan fmt lint build-all deps vet frontend dev-desktop
EXT_DIR = extension
.PHONY: build install clean test test-short run scan fmt lint build-all deps vet frontend dev-desktop ext ext-chrome ext-firefox ext-zip
frontend:
cd $(WEB_DIR) && $(NPM) ci && $(NPM) run build
@@ -63,5 +65,17 @@ build-all: frontend
GOOS=windows GOARCH=amd64 $(GO) build -o dist/$(BINARY)-windows-amd64.exe ./cmd/muyue/
GOOS=windows GOARCH=arm64 $(GO) build -o dist/$(BINARY)-windows-arm64.exe ./cmd/muyue/
ext:
cd $(EXT_DIR) && $(NPM) ci && $(NPM) run build && $(NPM) run build:firefox
ext-chrome:
cd $(EXT_DIR) && $(NPM) ci && $(NPM) run build
ext-firefox:
cd $(EXT_DIR) && $(NPM) ci && $(NPM) run build:firefox
ext-zip:
cd $(EXT_DIR) && $(NPM) ci && $(NPM) run zip && $(NPM) run zip:firefox
deps:
$(GO) mod tidy

View File

@@ -17,6 +17,45 @@ AI-powered development environment assistant by **La Légion de Muyue**.
- **i18n** — Full FR/EN support with keyboard layout awareness (AZERTY, QWERTY, QWERTZ)
- **4 themes** — Cyberpunk Red, Cyberpunk Pink, Midnight Blue, Matrix Green
## Browser Extension
Muyue ships a **browser extension** (Chrome, Edge, Firefox) that replaces the manual snippet injection for the Tests tab:
- **Auto-injects** the Muyue test client on every HTTP/HTTPS page — no more copy-paste
- **Captures console** errors/warnings in real-time
- **Native screenshots** via `captureVisibleTab` — pixel-perfect
- **Side Panel** (Chrome/Edge) and **Sidebar** (Firefox) for status monitoring
- **Badge** shows active session count or server status
### Install from source
```bash
cd extension
npm install
npm run build # Chrome/Edge → .output/chrome-mv3/
npm run build:firefox # Firefox → .output/firefox-mv2/
```
Then load the extension:
- **Chrome/Edge**: `chrome://extensions` → Developer mode → Load unpacked → select `extension/.output/chrome-mv3/`
- **Firefox**: `about:debugging#/runtime/this-firefox` → Load temporary Add-on → select any file in `extension/.output/firefox-mv2/`
### Download pre-built
Extension `.zip` files are attached to every [release](https://gitea.legion-muyue.fr/Muyue/MuyueWorkspace/releases):
- `muyue-extension-*-chrome.zip` — Chrome Web Store ready
- `muyue-extension-*-firefox.zip` — Firefox Add-ons ready
- `muyue-extension-*-sources.zip` — Required source for Firefox Add-ons review
### Development
```bash
cd extension
npm run dev # Chrome dev mode with HMR
npm run dev -- --browser firefox # Firefox dev mode
```
## Tech Stack
| Layer | Technology |
@@ -186,6 +225,10 @@ The Go backend serves 15 REST endpoints under `/api/`:
│ │ ├── styles/global.css # Full CSS theme system
│ │ └── themes/index.js # 4 themes with CSS variable injection
│ └── vite.config.js # Vite + dev proxy to :8095
├── extension/ # Browser extension (WXT, Chrome/Edge/Firefox)
│ ├── src/entrypoints/ # background, content, popup, sidepanel
│ ├── src/lib/ # config, page-rpc (shared logic)
│ └── src/styles/ # cyberpunk panel CSS
├── .gitea/workflows/ # CI/CD (PR check, beta, stable)
└── Makefile # build, test, lint, cross-compile
```

4
extension/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
node_modules/
.output/
.wxt/
*.zip

81
extension/README.md Normal file
View File

@@ -0,0 +1,81 @@
# Muyue Browser Extension
AI-powered browser testing & automation, connected to your [Muyue](https://github.com/muyue/muyue) desktop app.
## What it does
- **Auto-injects** the Muyue test client on every page — no more manual snippet copy-paste
- **Captures console** errors/warnings in real-time, sent to the AI Studio
- **Enables AI-driven testing**: click buttons, fill inputs, evaluate JS, take screenshots
- **Side Panel** (Chrome/Edge) and **Sidebar** (Firefox) for status monitoring
- **Native screenshots** via `chrome.tabs.captureVisibleTab` — pixel-perfect, no SVG hacks
- **URL change detection** via History API interception (survives SPA navigation)
- **Badge indicator**: shows connected session count or server status
## Install
### Chrome / Edge
1. Run `npm run build`
2. Open `chrome://extensions` → Enable **Developer mode**
3. Click **Load unpacked** → select `extension/.output/chrome-mv3/`
Or install the published extension from the Chrome Web Store.
### Firefox
1. Run `npm run build:firefox`
2. Open `about:debugging#/runtime/this-firefox`
3. Click **Load temporary Add-on** → select any file in `extension/.output/firefox-mv2/`
## Development
```bash
cd extension
npm install
npm run dev # Chrome dev mode with HMR
npm run dev -- --browser firefox # Firefox dev mode
```
## Build
```bash
npm run build # Chrome/Edge MV3 → .output/chrome-mv3/
npm run build:firefox # Firefox MV2 → .output/firefox-mv2/
npm run zip # Chrome .zip for Web Store
npm run zip:firefox # Firefox .zip + sources .zip
```
## Architecture
```
┌─────────────────────────────────────────────────────┐
│ Content Script (every HTTP/HTTPS page) │
│ - Console interception (log/warn/error) │
│ - RPC execution (click, type, eval, list) │
│ - URL change detection (History API + MutationObs) │
│ - WebSocket → Muyue server (same as snippet) │
└──────────────┬──────────────────────────────────────┘
│ chrome.runtime messaging
┌──────────────┴──────────────────────────────────────┐
│ Background Service Worker │
│ - Token management (GET /api/test/snippet) │
│ - Native screenshots (captureVisibleTab) │
│ - Badge updates (session count / server status) │
│ - chrome.alarms for periodic health checks │
└──────────────────────────────────────────────────────┘
┌──────────────────┐ ┌──────────────────┐
│ Popup │ │ Side Panel │
│ - Server status │ │ - Sessions list │
│ - Session count │ │ - Auto-refresh │
│ - Dashboard link │ │ - Dashboard link │
└──────────────────┘ └──────────────────┘
```
## Compatibility
| Browser | Manifest | Side Panel | Screenshots |
|---------|----------|------------|-------------|
| Chrome 89+ | MV3 | ✅ sidePanel API | ✅ captureVisibleTab |
| Edge 89+ | MV3 | ✅ sidePanel API | ✅ captureVisibleTab |
| Firefox | MV2 | ✅ sidebar API | ✅ tabs.captureVisibleTab |

4711
extension/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

16
extension/package.json Normal file
View File

@@ -0,0 +1,16 @@
{
"name": "muyue-extension",
"version": "0.8.0",
"private": true,
"type": "module",
"scripts": {
"dev": "wxt",
"build": "wxt build",
"build:firefox": "wxt build --browser firefox",
"zip": "wxt zip",
"zip:firefox": "wxt zip --browser firefox"
},
"dependencies": {
"wxt": "^0.20"
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 750 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 307 KiB

View File

@@ -0,0 +1,116 @@
import { fetchToken, fetchSessions, checkServerHealth, getServerUrl } from '../lib/config';
export default defineBackground(() => {
let token = null;
let wsUrl = null;
let serverOnline = false;
let errorCount = 0;
async function refreshToken() {
try {
const data = await fetchToken();
token = data.token;
wsUrl = data.wsUrl;
serverOnline = true;
return data;
} catch {
serverOnline = false;
token = null;
wsUrl = null;
return null;
}
}
async function updateBadge() {
try {
serverOnline = await checkServerHealth();
} catch {
serverOnline = false;
}
if (!serverOnline) {
chrome.action.setBadgeText({ text: '✕' });
chrome.action.setBadgeBackgroundColor({ color: '#ff6b6b' });
return;
}
try {
const sessions = await fetchSessions();
const count = sessions.length;
if (count > 0) {
chrome.action.setBadgeText({ text: String(count) });
chrome.action.setBadgeBackgroundColor({ color: '#3aaa61' });
} else {
chrome.action.setBadgeText({ text: '○' });
chrome.action.setBadgeBackgroundColor({ color: '#888' });
}
} catch {
chrome.action.setBadgeText({ text: '?' });
chrome.action.setBadgeBackgroundColor({ color: '#f5a623' });
}
}
async function handleScreenshot() {
try {
const dataUrl = await chrome.tabs.captureVisibleTab(null, {
format: 'png',
quality: 100,
});
return { ok: true, data_url: dataUrl };
} catch (e) {
return { ok: false, error: 'capture failed: ' + String(e) };
}
}
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
if (msg.type === 'get_state') {
getServerUrl().then((url) => {
sendResponse({
serverOnline,
token,
wsUrl,
errorCount,
serverUrl: url,
});
});
return true;
}
if (msg.type === 'get_token') {
refreshToken().then((data) => sendResponse(data));
return true;
}
if (msg.type === 'check_health') {
checkServerHealth().then((ok) => {
serverOnline = ok;
sendResponse({ online: ok });
});
return true;
}
if (msg.type === 'screenshot') {
handleScreenshot().then(sendResponse);
return true;
}
if (msg.type === 'refresh_badge') {
updateBadge();
return false;
}
if (msg.type === 'increment_errors') {
errorCount++;
return false;
}
return false;
});
chrome.alarms.create('muyue-badge', { periodInMinutes: 0.17 });
chrome.alarms.onAlarm.addListener((alarm) => {
if (alarm.name === 'muyue-badge') updateBadge();
});
updateBadge();
});

View File

@@ -0,0 +1,193 @@
import { dispatch } from '../lib/page-rpc';
export default defineContentScript({
matches: ['http://*/*', 'https://*/*'],
runAt: 'document_idle',
main() {
if (window.__muyueExtension) return;
window.__muyueExtension = true;
let ws = null;
let retryDelay = 0;
let token = null;
let wsBaseUrl = null;
const TAG = '[Muyue]';
function log(...args) {
console.log(TAG, ...args);
}
function send(obj) {
try {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(obj));
} catch {}
}
function reply(id, data) {
send({ type: 'reply', id, data });
}
function sendConsole(level, text) {
send({ type: 'console', level, text });
}
async function getToken() {
return new Promise((resolve) => {
chrome.runtime.sendMessage({ type: 'get_token' }, (response) => {
if (chrome.runtime.lastError) {
resolve(null);
return;
}
resolve(response);
});
});
}
function screenshotNative(params) {
return new Promise((resolve) => {
chrome.runtime.sendMessage({ type: 'screenshot', params }, (response) => {
if (chrome.runtime.lastError) {
resolve({ ok: false, error: String(chrome.runtime.lastError) });
return;
}
resolve(response || { ok: false, error: 'no response' });
});
});
}
async function connect() {
if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) {
return;
}
if (!token) {
const data = await getToken();
if (!data) {
retryDelay = Math.min(retryDelay + 1, 5);
setTimeout(connect, 1000 * retryDelay);
return;
}
token = data.token;
wsBaseUrl = data.wsUrl;
}
try {
const wsUrl = wsBaseUrl || `ws://127.0.0.1:8080/api/ws/browser-test?token=${token}`;
ws = new WebSocket(wsUrl);
} catch {
retryDelay = Math.min(retryDelay + 1, 5);
setTimeout(connect, 1000 * retryDelay);
return;
}
ws.onopen = () => {
retryDelay = 0;
send({ type: 'hello', url: location.href, title: document.title });
log('connected to Muyue server');
};
ws.onmessage = (ev) => {
let msg;
try { msg = JSON.parse(ev.data); } catch { return; }
if (msg.type === 'registered') {
log('session registered:', msg.session_id);
return;
}
if (msg.action) {
if (msg.action === 'screenshot') {
screenshotNative(msg.params || {}).then((r) => reply(msg.id, r));
return;
}
const result = dispatch(msg);
if (result && typeof result.then === 'function') {
result.then((r) => reply(msg.id, r));
} else {
reply(msg.id, result);
}
}
};
ws.onclose = () => {
retryDelay = Math.min(retryDelay + 1, 5);
setTimeout(connect, 500 * retryDelay);
};
ws.onerror = () => {};
}
['log', 'info', 'warn', 'error', 'debug'].forEach((lvl) => {
const orig = console[lvl];
console[lvl] = function () {
try {
const parts = Array.from(arguments).map((a) => {
if (typeof a === 'string') return a;
try { return JSON.stringify(a); } catch { return String(a); }
});
const text = parts.join(' ');
if (!text.startsWith(TAG)) {
sendConsole(lvl, text);
if (lvl === 'error') {
chrome.runtime.sendMessage({ type: 'increment_errors' }).catch(() => {});
}
}
} catch {}
return orig.apply(console, arguments);
};
});
window.addEventListener('error', (e) => {
sendConsole('error', 'window.onerror: ' + (e.message || 'unknown'));
chrome.runtime.sendMessage({ type: 'increment_errors' }).catch(() => {});
});
window.addEventListener('unhandledrejection', (e) => {
sendConsole('error', 'unhandledrejection: ' + String(e.reason));
chrome.runtime.sendMessage({ type: 'increment_errors' }).catch(() => {});
});
let lastUrl = location.href;
const urlObserver = new MutationObserver(() => {
if (location.href !== lastUrl) {
lastUrl = location.href;
send({ type: 'url_change', url: lastUrl });
}
});
urlObserver.observe(document.documentElement, { childList: true, subtree: true });
const origPushState = history.pushState;
history.pushState = function () {
origPushState.apply(this, arguments);
if (location.href !== lastUrl) {
lastUrl = location.href;
send({ type: 'url_change', url: lastUrl });
}
};
const origReplaceState = history.replaceState;
history.replaceState = function () {
origReplaceState.apply(this, arguments);
if (location.href !== lastUrl) {
lastUrl = location.href;
send({ type: 'url_change', url: lastUrl });
}
};
window.addEventListener('popstate', () => {
if (location.href !== lastUrl) {
lastUrl = location.href;
send({ type: 'url_change', url: lastUrl });
}
});
setInterval(() => {
if (location.href !== lastUrl) {
lastUrl = location.href;
send({ type: 'url_change', url: lastUrl });
}
}, 500);
connect();
},
});

View File

@@ -0,0 +1,55 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=320" />
</head>
<body>
<div class="panel">
<header>
<img src="/icon/32.png" alt="Muyue" />
<h1>Muyue</h1>
</header>
<div class="status-card">
<div class="status-row">
<span class="status-label">Server</span>
<span class="status-value" id="server-status">
<span class="dot dot-yellow"></span>Checking…
</span>
</div>
<div class="status-row">
<span class="status-label">Active sessions</span>
<span class="status-value" id="session-count"></span>
</div>
<div class="status-row">
<span class="status-label">Console errors</span>
<span class="status-value" id="error-count">0</span>
</div>
</div>
<div class="actions">
<a id="btn-dashboard" href="#" class="btn btn-primary" target="_blank">
Open Dashboard
</a>
<button id="btn-sidepanel" class="btn">
Open Side Panel
</button>
</div>
<div class="settings-section">
<label>Server URL</label>
<div class="input-row">
<input type="text" id="server-url" placeholder="http://127.0.0.1:8080" />
<button id="btn-save-url">Save</button>
</div>
</div>
<div class="footer">
<span>Muyue</span> browser extension v0.1.0
</div>
</div>
<script src="./main.js" type="module"></script>
</body>
</html>

View File

@@ -0,0 +1,54 @@
import '../../styles/panel.css';
import { getServerUrl, setServerUrl, fetchSessions } from '../../lib/config';
const $serverStatus = document.getElementById('server-status');
const $sessionCount = document.getElementById('session-count');
const $errorCount = document.getElementById('error-count');
const $btnDashboard = document.getElementById('btn-dashboard');
const $btnSidepanel = document.getElementById('btn-sidepanel');
const $serverUrl = document.getElementById('server-url');
const $btnSaveUrl = document.getElementById('btn-save-url');
function dot(color) {
return `<span class="dot dot-${color}"></span>`;
}
async function refresh() {
const url = await getServerUrl();
$serverUrl.value = url;
$btnDashboard.href = url;
try {
const sessions = await fetchSessions();
$serverStatus.innerHTML = `${dot('green')} Online`;
$sessionCount.textContent = sessions.length;
} catch {
$serverStatus.innerHTML = `${dot('red')} Offline`;
$sessionCount.textContent = '—';
}
chrome.runtime.sendMessage({ type: 'get_state' }, (state) => {
if (chrome.runtime.lastError || !state) return;
$errorCount.textContent = state.errorCount || 0;
});
}
$btnSaveUrl.addEventListener('click', async () => {
const url = $serverUrl.value.trim().replace(/\/$/, '');
if (url) {
await setServerUrl(url);
refresh();
}
});
$btnSidepanel.addEventListener('click', async () => {
try {
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
if (tab) {
chrome.sidePanel.open({ tabId: tab.id });
window.close();
}
} catch {}
});
refresh();

View File

@@ -0,0 +1,54 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=100%" />
</head>
<body>
<div class="panel">
<header>
<img src="/icon/32.png" alt="Muyue" />
<h1>Muyue Side Panel</h1>
</header>
<div class="status-card">
<div class="status-row">
<span class="status-label">Server</span>
<span class="status-value" id="server-status">
<span class="dot dot-yellow"></span>Checking…
</span>
</div>
<div class="status-row">
<span class="status-label">Active sessions</span>
<span class="status-value" id="session-count"></span>
</div>
<div class="status-row">
<span class="status-label">Console errors</span>
<span class="status-value" id="error-count">0</span>
</div>
</div>
<div id="sessions-list"></div>
<div class="actions">
<a id="btn-dashboard" href="#" class="btn btn-primary" target="_blank">
Open Dashboard
</a>
</div>
<div class="settings-section">
<label>Server URL</label>
<div class="input-row">
<input type="text" id="server-url" placeholder="http://127.0.0.1:8080" />
<button id="btn-save-url">Save</button>
</div>
</div>
<div class="footer">
<span>Muyue</span> browser extension v0.1.0
</div>
</div>
<script src="./main.js" type="module"></script>
</body>
</html>

View File

@@ -0,0 +1,72 @@
import '../../styles/panel.css';
import { getServerUrl, setServerUrl, fetchSessions } from '../../lib/config';
const $serverStatus = document.getElementById('server-status');
const $sessionCount = document.getElementById('session-count');
const $errorCount = document.getElementById('error-count');
const $sessionsList = document.getElementById('sessions-list');
const $btnDashboard = document.getElementById('btn-dashboard');
const $serverUrl = document.getElementById('server-url');
const $btnSaveUrl = document.getElementById('btn-save-url');
function dot(color) {
return `<span class="dot dot-${color}"></span>`;
}
function renderSessions(sessions) {
if (sessions.length === 0) {
$sessionsList.innerHTML = '';
return;
}
$sessionsList.innerHTML = `
<div class="status-card" style="margin-top:12px">
<div style="font-size:11px;color:var(--text-secondary);margin-bottom:8px;text-transform:uppercase;letter-spacing:0.5px">
Connected tabs
</div>
${sessions.map((s) => `
<div class="status-row">
<span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:200px" title="${s.url}">
${s.title || s.url || s.id}
</span>
<span style="font-size:10px;color:var(--text-secondary);font-family:var(--font-mono)">
${s.id.slice(0, 8)}
</span>
</div>
`).join('')}
</div>
`;
}
async function refresh() {
const url = await getServerUrl();
$serverUrl.value = url;
$btnDashboard.href = url;
try {
const sessions = await fetchSessions();
$serverStatus.innerHTML = `${dot('green')} Online`;
$sessionCount.textContent = sessions.length;
renderSessions(sessions);
} catch {
$serverStatus.innerHTML = `${dot('red')} Offline`;
$sessionCount.textContent = '—';
$sessionsList.innerHTML = '';
}
chrome.runtime.sendMessage({ type: 'get_state' }, (state) => {
if (chrome.runtime.lastError || !state) return;
$errorCount.textContent = state.errorCount || 0;
});
}
$btnSaveUrl.addEventListener('click', async () => {
const url = $serverUrl.value.trim().replace(/\/$/, '');
if (url) {
await setServerUrl(url);
refresh();
}
});
refresh();
setInterval(refresh, 5000);

View File

@@ -0,0 +1,56 @@
const DEFAULT_PORT = 8080;
const DEFAULT_HOST = '127.0.0.1';
const DEFAULT_URL = `http://${DEFAULT_HOST}:${DEFAULT_PORT}`;
function isServiceWorker() {
return typeof ServiceWorkerGlobalScope !== 'undefined' && self instanceof ServiceWorkerGlobalScope;
}
export async function getServerUrl() {
if (isServiceWorker()) {
const result = await chrome.storage.local.get('muyue_server_url');
return result.muyue_server_url || DEFAULT_URL;
}
const stored = localStorage.getItem('muyue_server_url');
return stored || DEFAULT_URL;
}
export async function setServerUrl(url) {
if (isServiceWorker()) {
await chrome.storage.local.set({ muyue_server_url: url });
} else {
localStorage.setItem('muyue_server_url', url);
}
}
export async function buildWsUrl(token) {
const base = await getServerUrl();
const wsBase = base.replace(/^http/, 'ws');
return `${wsBase}/api/ws/browser-test?token=${encodeURIComponent(token)}`;
}
export async function fetchToken() {
const base = await getServerUrl();
const res = await fetch(`${base}/api/test/snippet`);
if (!res.ok) throw new Error(`Server returned ${res.status}`);
const data = await res.json();
return { token: data.token, wsUrl: data.ws_url, expiresIn: data.expires_in };
}
export async function fetchSessions() {
const base = await getServerUrl();
const res = await fetch(`${base}/api/test/sessions`);
if (!res.ok) throw new Error(`Server returned ${res.status}`);
const data = await res.json();
return data.sessions || [];
}
export async function checkServerHealth() {
try {
const base = await getServerUrl();
const res = await fetch(`${base}/api/info`, { signal: AbortSignal.timeout(3000) });
return res.ok;
} catch {
return false;
}
}

View File

@@ -0,0 +1,113 @@
let lastList = [];
function safeText(el) {
let t = (el.innerText || el.textContent || '').trim();
if (t.length > 80) t = t.slice(0, 80) + '…';
return t;
}
function describe(el) {
let sel = el.id ? '#' + el.id : el.tagName.toLowerCase();
if (!el.id && el.className && typeof el.className === 'string') {
sel += '.' + el.className.trim().split(/\s+/).slice(0, 2).join('.');
}
const label = el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('name') || '';
return {
tag: el.tagName.toLowerCase(),
selector: sel,
text: safeText(el),
label,
type: el.getAttribute('type') || '',
disabled: !!el.disabled,
};
}
export function listClickables() {
const els = Array.from(
document.querySelectorAll(
'button, a[href], input[type=submit], input[type=button], [role=button], [onclick]'
)
);
lastList = els.filter((e) => {
const r = e.getBoundingClientRect();
return r.width > 0 && r.height > 0;
});
return lastList.map((el, i) => {
const d = describe(el);
d.index = i;
return d;
});
}
export function clickElement(params) {
let el;
if (params.selector) el = document.querySelector(params.selector);
else if (typeof params.index === 'number') el = lastList[params.index];
if (!el) return { ok: false, error: 'element not found' };
if (el.disabled) return { ok: false, error: 'element is disabled' };
try {
el.scrollIntoView({ block: 'center' });
el.click();
return { ok: true };
} catch (e) {
return { ok: false, error: String(e) };
}
}
export function typeText(params) {
let el;
if (params.selector) el = document.querySelector(params.selector);
else if (typeof params.index === 'number') el = lastList[params.index];
if (!el) return { ok: false, error: 'element not found' };
const proto = Object.getPrototypeOf(el);
const setter = Object.getOwnPropertyDescriptor(proto, 'value');
try {
if (setter && setter.set) setter.set.call(el, params.text || '');
else el.value = params.text || '';
} catch {
el.value = params.text || '';
}
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return { ok: true };
}
export function evalExpr(params) {
try {
const r = (0, eval)(params.expr);
return { ok: true, value: serialize(r) };
} catch (e) {
return { ok: false, error: String(e) };
}
}
export function currentUrl() {
return { url: location.href, title: document.title };
}
function serialize(v) {
if (v === undefined) return 'undefined';
try {
return JSON.parse(JSON.stringify(v));
} catch {
return String(v);
}
}
export function dispatch(msg) {
const p = msg.params || {};
switch (msg.action) {
case 'list_clickables':
return listClickables();
case 'click':
return clickElement(p);
case 'eval':
return evalExpr(p);
case 'current_url':
return currentUrl();
case 'type':
return typeText(p);
default:
return { ok: false, error: 'unknown action: ' + msg.action };
}
}

View File

@@ -0,0 +1,211 @@
:root {
--bg-primary: #0a0a0f;
--bg-secondary: #12121a;
--bg-tertiary: rgba(255, 255, 255, 0.05);
--border: rgba(255, 255, 255, 0.1);
--text-primary: #e8e8f0;
--text-secondary: #9999aa;
--accent: #ff4757;
--accent-dim: rgba(255, 71, 87, 0.15);
--accent-glow: rgba(255, 71, 87, 0.4);
--green: #3aaa61;
--yellow: #f5a623;
--red: #ff6b6b;
--font-mono: 'JetBrains Mono', ui-monospace, monospace;
--font-sans: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
}
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: var(--font-sans);
background: var(--bg-primary);
color: var(--text-primary);
font-size: 13px;
line-height: 1.4;
}
.panel {
width: 320px;
padding: 16px;
}
header {
display: flex;
align-items: center;
gap: 10px;
margin-bottom: 16px;
padding-bottom: 12px;
border-bottom: 1px solid var(--border);
}
header img {
width: 28px;
height: 28px;
}
header h1 {
font-size: 16px;
font-weight: 600;
letter-spacing: -0.3px;
}
.status-card {
background: var(--bg-secondary);
border: 1px solid var(--border);
border-radius: 8px;
padding: 12px;
margin-bottom: 12px;
}
.status-row {
display: flex;
align-items: center;
justify-content: space-between;
padding: 6px 0;
}
.status-row + .status-row {
border-top: 1px solid var(--border);
margin-top: 6px;
padding-top: 10px;
}
.status-label {
color: var(--text-secondary);
font-size: 12px;
}
.status-value {
font-weight: 500;
font-size: 12px;
}
.dot {
display: inline-block;
width: 8px;
height: 8px;
border-radius: 50%;
margin-right: 6px;
}
.dot-green { background: var(--green); box-shadow: 0 0 6px var(--green); }
.dot-red { background: var(--red); box-shadow: 0 0 6px var(--red); }
.dot-yellow { background: var(--yellow); box-shadow: 0 0 6px var(--yellow); }
.actions {
display: flex;
flex-direction: column;
gap: 8px;
margin-top: 12px;
}
.btn {
display: flex;
align-items: center;
justify-content: center;
gap: 6px;
padding: 9px 14px;
border-radius: 6px;
border: 1px solid var(--border);
background: var(--bg-tertiary);
color: var(--text-primary);
font-size: 12px;
font-weight: 500;
cursor: pointer;
transition: all 0.15s ease;
text-decoration: none;
}
.btn:hover {
background: var(--accent-dim);
border-color: var(--accent);
}
.btn-primary {
background: var(--accent);
border-color: var(--accent);
color: #fff;
}
.btn-primary:hover {
background: #e8414f;
box-shadow: 0 0 12px var(--accent-glow);
}
.settings-section {
margin-top: 12px;
padding-top: 12px;
border-top: 1px solid var(--border);
}
.settings-section label {
display: block;
color: var(--text-secondary);
font-size: 11px;
margin-bottom: 4px;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.input-row {
display: flex;
gap: 6px;
}
.input-row input {
flex: 1;
background: var(--bg-secondary);
border: 1px solid var(--border);
border-radius: 4px;
padding: 6px 8px;
color: var(--text-primary);
font-size: 12px;
font-family: var(--font-mono);
outline: none;
}
.input-row input:focus {
border-color: var(--accent);
}
.input-row button {
padding: 6px 10px;
border-radius: 4px;
border: 1px solid var(--border);
background: var(--bg-tertiary);
color: var(--text-primary);
cursor: pointer;
font-size: 11px;
}
.input-row button:hover {
background: var(--accent-dim);
border-color: var(--accent);
}
.footer {
margin-top: 12px;
padding-top: 10px;
border-top: 1px solid var(--border);
text-align: center;
color: var(--text-secondary);
font-size: 10px;
}
.footer span {
color: var(--accent);
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.5; }
}
.loading {
animation: pulse 1.5s ease-in-out infinite;
}

30
extension/wxt.config.js Normal file
View File

@@ -0,0 +1,30 @@
import { defineConfig } from 'wxt';
export default defineConfig({
srcDir: 'src',
suppressWarnings: {
firefoxDataCollection: true,
},
manifest: {
name: 'Muyue',
description: 'AI-powered browser testing & automation — connected to your Muyue desktop app',
permissions: [
'storage',
'activeTab',
'tabs',
'sidePanel',
'scripting',
'notifications',
],
host_permissions: ['http://127.0.0.1:*/*', 'http://localhost:*/*'],
action: {
default_icon: {
16: 'icon/16.png',
32: 'icon/32.png',
},
},
side_panel: {
default_path: 'sidepanel.html',
},
},
});

View File

@@ -66,23 +66,52 @@ Muyue gère :
| **browser_test** | Piloter un onglet de navigateur de l'utilisateur (clic, eval, lecture console) — voir `<browser_test_strategy>` ci-dessous |
<browser_test_strategy>
Quand l'utilisateur demande de **tester** une UI / une page (ses boutons, ses formulaires, son comportement), utilise `browser_test`. La page cible doit déjà être connectée via le snippet de l'onglet "Tests" — sinon, l'outil te le dira et tu demandes à l'utilisateur de coller le snippet.
Quand l'utilisateur demande de **tester** une UI (ses boutons, ses formulaires, son comportement), utilise `browser_test`. La page cible doit être connectée via le snippet de l'onglet **Tests** — sinon, l'outil te le dira et tu demandes à l'utilisateur de coller le snippet (le même token reste valide même après reload : si la connexion est perdue, l'utilisateur n'a qu'à re-coller).
Boucle recommandée :
## Règle d'or — économise les appels d'outils
1. `browser_test` action `summary` — voir l'URL, le titre et les dernières erreurs console déjà présentes.
2. `browser_test` action `list_clickables` — récupérer la liste indexée des boutons / liens / inputs cliquables.
3. Pour chaque cible : `browser_test` action `click` (avec `index` ou `selector`).
4. Immédiatement après chaque clic, **regarde le `console_delta` retourné** : c'est la liste des messages console émis pendant le clic. `level: "error"` = bouton cassé.
5. Vérifie aussi `current_url` retourné — un changement d'URL inattendu peut signaler un bug.
6. Si l'élément ouvre un dialog ou modifie le DOM, refais `list_clickables` pour découvrir les nouveaux éléments.
7. Pour les inputs : utilise `type` avant `click` sur le bouton de soumission.
8. À la fin, fournis un **rapport** structuré : ✓ boutons OK / ✗ boutons cassés (avec le message d'erreur exact) / ⚠ boutons disabled ou non trouvés.
**N'appelle PAS `list_clickables` après chaque clic.** C'est l'erreur n°1 qui fait exploser ta boucle (150+ appels pour 5 actions humaines). La liste change rarement et chaque appel renvoie ~30-100 éléments.
Stratégie efficace :
1. **Au début** : `summary` (URL + console + 20 lignes) → `list_clickables` (UNE FOIS, mémorise les index pertinents pour ta tâche).
2. **Pendant** : clique par `index`. Lis le `console_delta` retourné après chaque clic.
3. **Re-list seulement si** :
- le `current_url` retourné change ET la nouvelle page est inconnue,
- OU un clic ouvre un dialog / nouveau composant que tu dois inspecter,
- OU `click` retourne `element not found` (DOM a muté).
4. Pour les pages SPA qui rechargent côté URL mais pas le DOM, vérifie d'abord avec `eval document.querySelectorAll('button').length` — si stable, ne re-liste pas.
5. Si tu te sens bloqué, **ne boucle pas en aveugle**. Fais 1 `summary`, 1 `eval` ciblé, et demande de l'aide à l'utilisateur. Mieux vaut 5 appels et une question qu'une boucle de 50 appels.
## Actions disponibles
| Action | Quand l'utiliser |
|---|---|
| `summary` | État de la page (URL, titre, 20 dernières lignes console). Appel **bon marché**. |
| `list_clickables` | Liste indexée des boutons/liens/inputs visibles. **Appel cher** (~50+ items) — utilise avec parcimonie. |
| `click` (par `index` de préférence) | Clique. Retourne `console_delta` + `current_url`. |
| `type` | Remplit un input (par `selector` ou `index`). Toujours suivi d'un `click` sur le bouton submit. |
| `eval` | JS arbitraire. Idéal pour des questions ciblées (`document.title`, `document.querySelectorAll(X).length`, etc.) au lieu de `list_clickables` complet. |
| `current_url` | URL+titre. Très bon marché. |
| `wait` | Pause 200-500 ms après une action async (transition / fetch). |
| `console` | N dernières lignes console (default 50). Pour debug post-incident. |
| `screenshot` | Capture viewport (ou `selector`) et sauve dans `~/.muyue/screenshots/<filename>.png`. Utilise `filename` pour nommer ; sinon timestamp. Best-effort (CSS externe / images peuvent ne pas apparaître). |
## Rapport final
Quand tous les tests sont terminés, fournis un rapport **structuré et bref** :
```
✓ Boutons OK : <liste des labels>
✗ Boutons cassés : <label> — <message d'erreur exact du console_delta>
⚠ Bloqués : <label> — <pourquoi> (disabled, non trouvé, etc.)
📸 Captures : <chemins relatifs sous ~/.muyue/screenshots/>
```
Astuces :
- Préfère cliquer **par `index`** que par sélecteur — le sélecteur change avec le DOM, l'index reste stable jusqu'au prochain `list_clickables`.
- Entre deux actions sensibles, `wait` 200-500 ms si la page a des transitions / fetches asynchrones.
- Clique **par index** ; le sélecteur peut changer avec le DOM, l'index reste stable jusqu'au prochain `list_clickables`.
- N'utilise jamais `eval` pour cliquer si `click` suffit.
- Si la page se recharge (`current_url` change ou la connexion tombe), demande à l'utilisateur de recoller le snippet — le même token marche.
</browser_test_strategy>
<tool_strategy>

View File

@@ -12,10 +12,12 @@ package api
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"sync"
"time"
@@ -24,8 +26,20 @@ import (
"github.com/muyue/muyue/internal/agent"
)
// thin os wrappers (kept here so saveScreenshot stays independent of any
// existing helper file's evolution)
func osUserHomeDir() (string, error) { return os.UserHomeDir() }
func mkdirAll(p string, m os.FileMode) error { return os.MkdirAll(p, m) }
func writeFile(p string, b []byte, m os.FileMode) error { return os.WriteFile(p, b, m) }
func base64StdDecode(s string) ([]byte, error) { return base64.StdEncoding.DecodeString(s) }
const (
browserTestTokenTTL = 5 * time.Minute
// browserTestTokenTTL is a sliding window: every successful WS connect
// using the token resets it. So the user re-pasting the snippet after a
// page reload / navigation seamlessly resumes (same token, same session
// continuation in the AI's view), as long as no more than this gap of
// inactivity occurs.
browserTestTokenTTL = 60 * time.Minute
browserTestCommandTTL = 30 * time.Second
browserTestConsoleMax = 200
browserTestSessionsMax = 16
@@ -86,7 +100,11 @@ func (s *BrowserTestStore) IssueToken() string {
return tok
}
// ConsumeToken validates and removes a token in one step.
// ConsumeToken validates a token. Tokens are no longer single-use:
// the test snippet re-establishes the WS after every page reload /
// navigation, so the same token must work multiple times. We slide the
// expiration on each successful use so a long active test session keeps
// the token alive.
func (s *BrowserTestStore) ConsumeToken(tok string) bool {
s.tokensMu.Lock()
defer s.tokensMu.Unlock()
@@ -94,8 +112,12 @@ func (s *BrowserTestStore) ConsumeToken(tok string) bool {
if !ok {
return false
}
if time.Since(t) > browserTestTokenTTL {
delete(s.tokens, tok)
return time.Since(t) <= browserTestTokenTTL
return false
}
s.tokens[tok] = time.Now() // sliding refresh
return true
}
// Register inserts a new session, evicting the oldest if at capacity.
@@ -377,14 +399,15 @@ func (s *Server) handleBrowserTestWS(w http.ResponseWriter, r *http.Request) {
// BrowserTestParams is the schema exposed to the AI for the browser_test tool.
type BrowserTestParams struct {
Action string `json:"action" description:"One of: list_clickables, click, eval, console, current_url, wait, type, summary"`
Action string `json:"action" description:"One of: list_clickables, click, eval, console, current_url, wait, type, summary, screenshot"`
SessionID string `json:"session_id,omitempty" description:"Browser session id (optional, defaults to most recent)"`
Selector string `json:"selector,omitempty" description:"CSS selector for click/type actions"`
Selector string `json:"selector,omitempty" description:"CSS selector for click/type/screenshot actions (screenshot defaults to whole viewport when omitted)"`
Index int `json:"index,omitempty" description:"Alternative to selector: index into the last list_clickables result (0-based)"`
Expr string `json:"expr,omitempty" description:"JS expression to evaluate (eval action only)"`
Text string `json:"text,omitempty" description:"Text to type (type action only)"`
WaitMs int `json:"wait_ms,omitempty" description:"Milliseconds to wait (wait action only, max 5000)"`
Tail int `json:"tail,omitempty" description:"Console action: how many recent lines to return (default 50, max 200)"`
Filename string `json:"filename,omitempty" description:"Screenshot action: optional file name (no path, no extension); defaults to a timestamp"`
}
// RegisterBrowserTestTool wires the agent tool against a session store.
@@ -401,7 +424,7 @@ func RegisterBrowserTestTool(reg *agent.Registry, store *BrowserTestStore) error
switch action {
case "":
return agent.TextErrorResponse("action is required"), nil
case "list_clickables", "click", "eval", "current_url", "type":
case "list_clickables", "click", "eval", "current_url", "type", "screenshot":
case "console", "summary", "wait":
default:
return agent.TextErrorResponse("unknown action: " + p.Action), nil
@@ -479,6 +502,23 @@ func RegisterBrowserTestTool(reg *agent.Registry, store *BrowserTestStore) error
return agent.TextErrorResponse(err.Error()), nil
}
// Screenshot post-processing: snippet returns a base64 data URL;
// decode and write to ~/.muyue/screenshots/<filename>.png so the
// AI can reference an on-disk path rather than streaming megabytes
// of base64 back through its context.
if action == "screenshot" {
saved, perr := saveScreenshot(payload, p.Filename)
if perr != nil {
return agent.TextErrorResponse("screenshot save: " + perr.Error()), nil
}
out, _ := json.MarshalIndent(map[string]interface{}{
"action": "screenshot",
"saved_to": saved,
"current_url": sess.URL,
}, "", " ")
return agent.TextResponse(string(out)), nil
}
// Console delta: messages logged during this command.
post := sess.SnapshotConsole()
var delta []ConsoleEntry
@@ -504,14 +544,21 @@ func RegisterBrowserTestTool(reg *agent.Registry, store *BrowserTestStore) error
// Snippet generator ----------------------------------------------------------
func buildBrowserTestSnippet(wsURL string) string {
// Note: this is the JS injected into the user's target page. It opens the
// WS, hooks console, and dispatches commands. Kept terse on purpose.
// Inline JS injected into the user's target page. Responsibilities:
// - open the WS, with auto-reconnect (exponential backoff capped at 5s)
// - hook console.log/info/warn/error/debug + window.onerror + unhandledrejection
// - dispatch RPC commands: list_clickables, click, type, eval, current_url, screenshot
// - re-establish WS on transient close (network blip, server restart, etc.)
//
// Across full page navigation / reload the JS context is destroyed —
// no JS-only mechanism can survive that. The token is reusable (sliding
// 60-min TTL server-side), so the user just re-pastes the same snippet
// from the Tests tab to resume.
return `(function(){
if (window.__muyueTestRunner) { console.log('[Muyue] runner already attached'); return; }
var WS_URL = ` + jsString(wsURL) + `;
var ws = new WebSocket(WS_URL);
var lastList = [];
function send(obj){ try{ ws.send(JSON.stringify(obj)); }catch(e){} }
var ws = null, lastList = [], retry = 0;
function send(obj){ try{ if (ws && ws.readyState === 1) ws.send(JSON.stringify(obj)); }catch(e){} }
function reply(id, data){ send({type:'reply', id:id, data:data}); }
function safeText(el){
var t = (el.innerText || el.textContent || '').trim();
@@ -537,6 +584,36 @@ func buildBrowserTestSnippet(wsURL string) string {
try { el.scrollIntoView({block:'center'}); el.click(); return { ok:true }; }
catch(e){ return { ok:false, error:String(e) }; }
}
// Best-effort viewport screenshot via SVG foreignObject — works on most
// pages, but external CSS / images / iframes won't be inlined. Returns a
// base64 PNG data URL the server will save to disk.
function screenshot(p){
return new Promise(function(resolve){
try {
var w = Math.max(document.documentElement.clientWidth, 1024);
var h = Math.max(window.innerHeight, 768);
var node = (p && p.selector) ? document.querySelector(p.selector) : document.documentElement;
if (!node) { resolve({ ok:false, error:'selector not found' }); return; }
var rect = node.getBoundingClientRect();
if (node === document.documentElement) { rect = { width:w, height:h }; }
var clone = node.cloneNode(true);
var ser = new XMLSerializer().serializeToString(clone);
var svg = '<svg xmlns="http://www.w3.org/2000/svg" width="'+Math.round(rect.width)+'" height="'+Math.round(rect.height)+'">' +
'<foreignObject width="100%" height="100%"><div xmlns="http://www.w3.org/1999/xhtml" style="background:white">' + ser + '</div></foreignObject></svg>';
var img = new Image();
img.onload = function(){
try {
var c = document.createElement('canvas');
c.width = Math.round(rect.width); c.height = Math.round(rect.height);
c.getContext('2d').drawImage(img, 0, 0);
resolve({ ok:true, data_url: c.toDataURL('image/png'), width: c.width, height: c.height });
} catch(e){ resolve({ ok:false, error:'canvas: '+String(e) }); }
};
img.onerror = function(){ resolve({ ok:false, error:'image load failed (CSP or invalid SVG)' }); };
img.src = 'data:image/svg+xml;charset=utf-8,' + encodeURIComponent(svg);
} catch(e){ resolve({ ok:false, error:String(e) }); }
});
}
function dispatch(msg){
var p = msg.params || {};
switch(msg.action){
@@ -563,6 +640,8 @@ func buildBrowserTestSnippet(wsURL string) string {
el.dispatchEvent(new Event('change', {bubbles:true}));
return { ok:true };
}
case 'screenshot':
return screenshot(p);
}
return { ok:false, error:'unknown action' };
}
@@ -594,15 +673,29 @@ func buildBrowserTestSnippet(wsURL string) string {
setInterval(function(){
if (location.href !== lastUrl){ lastUrl = location.href; send({type:'url_change', url: lastUrl}); }
}, 500);
ws.onopen = function(){ send({type:'hello', url: location.href, title: document.title}); };
function connect(){
ws = new WebSocket(WS_URL);
ws.onopen = function(){ retry = 0; send({type:'hello', url: location.href, title: document.title}); };
ws.onmessage = function(ev){
try { var msg = JSON.parse(ev.data); }
catch(e){ return; }
try { var msg = JSON.parse(ev.data); } catch(e){ return; }
if (msg.type === 'registered') { console.log('[Muyue] connected — session', msg.session_id); return; }
if (msg.action) reply(msg.id, dispatch(msg));
if (msg.action) {
var out = dispatch(msg);
if (out && typeof out.then === 'function') { out.then(function(r){ reply(msg.id, r); }); }
else { reply(msg.id, out); }
}
};
ws.onclose = function(){ console.log('[Muyue] runner disconnected'); window.__muyueTestRunner = null; };
window.__muyueTestRunner = { ws: ws, list: list };
ws.onclose = function(){
// Same-page transient disconnect → reconnect with backoff up to ~5s.
// Full navigation kills the JS context entirely — this never runs in
// that case; the user re-pastes the snippet (same token works).
retry = Math.min(retry + 1, 5);
setTimeout(connect, 500 * retry);
};
ws.onerror = function(){ /* onclose will fire next */ };
}
connect();
window.__muyueTestRunner = { reconnect: connect, list: list };
})();`
}
@@ -610,3 +703,70 @@ func jsString(s string) string {
b, _ := json.Marshal(s)
return string(b)
}
// saveScreenshot decodes the base64 PNG returned by the snippet's
// screenshot action and writes it to ~/.muyue/screenshots/<name>.png.
// Returns the absolute path saved, or an error.
func saveScreenshot(replyPayload json.RawMessage, requestedName string) (string, error) {
var reply struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DataURL string `json:"data_url,omitempty"`
}
if err := json.Unmarshal(replyPayload, &reply); err != nil {
return "", fmt.Errorf("invalid reply: %w", err)
}
if !reply.OK {
if reply.Error != "" {
return "", fmt.Errorf("snippet: %s", reply.Error)
}
return "", fmt.Errorf("snippet returned ok=false")
}
const prefix = "data:image/png;base64,"
if !strings.HasPrefix(reply.DataURL, prefix) {
return "", fmt.Errorf("unexpected data URL prefix")
}
raw, err := base64StdDecode(reply.DataURL[len(prefix):])
if err != nil {
return "", fmt.Errorf("base64: %w", err)
}
dir, err := screenshotDir()
if err != nil {
return "", err
}
name := sanitizeFilename(requestedName)
if name == "" {
name = time.Now().Format("20060102-150405")
}
path := dir + "/" + name + ".png"
if err := writeFile(path, raw, 0644); err != nil {
return "", err
}
return path, nil
}
func screenshotDir() (string, error) {
home, err := osUserHomeDir()
if err != nil {
return "", err
}
dir := home + "/.muyue/screenshots"
if err := mkdirAll(dir, 0755); err != nil {
return "", err
}
return dir, nil
}
// sanitizeFilename keeps a safe subset (letters / digits / _ / - / .) so
// the user-supplied name cannot escape the screenshots directory.
func sanitizeFilename(s string) string {
var b strings.Builder
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9',
r == '_', r == '-', r == '.':
b.WriteRune(r)
}
}
return b.String()
}

View File

@@ -80,9 +80,15 @@ func startConptySession(cmd *exec.Cmd) (termSession, error) {
windows.CloseHandle(outRead)
return nil, fmt.Errorf("NewProcThreadAttributeList: %w", err)
}
// PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE is a quirk of the Win32 API: lpValue
// is the HPCON *value* (cast to PVOID), not a pointer to the handle. If
// we pass &hPC the kernel reads garbage, the PC attribute is silently
// ignored, and cmd/pwsh get their own external console window — which is
// exactly the regression v0.7.6 introduced. The cbSize stays the size of
// the handle (8 bytes on amd64). Reference: Microsoft EchoCon sample.
if err := attrList.Update(
procThreadAttributePseudoconsole,
unsafe.Pointer(&hPC),
unsafe.Pointer(uintptr(hPC)),
unsafe.Sizeof(hPC),
); err != nil {
attrList.Delete()

View File

@@ -7,7 +7,7 @@ import (
const (
Name = "muyue"
Version = "0.7.7"
Version = "0.8.0"
Author = "La Légion de Muyue"
)