✨ chore(eval): add run Harbor Skill (#19625)

* ✨ feat(eval): add local Harbor harness

* ✨ feat(eval): support local and cloud Harbor targets

* 🐛 fix(eval): expose npm lh binary to Harbor shells

* 🐛 fix(eval): harden Harbor bootstrap and runner

.agents/acceptance/scripts/init-dev-env.sh

@@ -10,6 +10,8 @@
 # Existing local config always wins.
 #
 # Usage:
+#   init-dev-env.sh [--env-file <file>] <command>
+#   --env-file is accepted only by env and seed-user for isolated harnesses.
 #   init-dev-env.sh env              # print shell exports
 #   init-dev-env.sh write [file]     # write a source-able env file
 #   init-dev-env.sh setup-db         # start local Postgres/Redis and run migrations
@@ -33,6 +35,26 @@ set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
 ROOT_ENV_FILE="$REPO_ROOT/.env"
+EXPLICIT_ENV_FILE=""
+
+if [[ "${1:-}" == "--env-file" ]]; then
+  if [[ -z "${2:-}" || ! -f "$2" ]]; then
+    printf 'ERROR: --env-file requires an existing file.\n' >&2
+    exit 2
+  fi
+
+  EXPLICIT_ENV_FILE="$(cd "$(dirname "$2")" && pwd -P)/$(basename "$2")"
+  if [[ -e "$ROOT_ENV_FILE" && "$EXPLICIT_ENV_FILE" -ef "$ROOT_ENV_FILE" ]]; then
+    printf 'ERROR: --env-file cannot point to the repository root .env.\n' >&2
+    exit 2
+  fi
+
+  set -a
+  # shellcheck disable=SC1090
+  source "$EXPLICIT_ENV_FILE"
+  set +a
+  shift 2
+fi
 
 # Resolve the workspace root the SAME way test-env.sh does, so both scripts
 # read/write the ports file (and other .records artifacts) at the same path.
@@ -159,7 +181,7 @@ _qstash_reachable() {
 }
 
 guard_no_root_env() {
-  if [[ -f "$ROOT_ENV_FILE" ]]; then
+  if [[ -f "$ROOT_ENV_FILE" && -z "$EXPLICIT_ENV_FILE" ]]; then
     bad "root .env exists: $ROOT_ENV_FILE"
     note "Use the existing local configuration instead of init-dev-env.sh."
     note "Start normally from repo root, e.g. pnpm run dev:next or bun run dev."
@@ -789,6 +811,11 @@ usage() {
 
 COMMAND="${1:-status}"
 
+if [[ -n "$EXPLICIT_ENV_FILE" && "$COMMAND" != "env" && "$COMMAND" != "seed-user" ]]; then
+  bad "--env-file is supported only by env and seed-user"
+  exit 2
+fi
+
 case "$COMMAND" in
   help|-h|--help) usage; exit 0 ;;
   *) guard_no_root_env ;;

.agents/skills/run-eval-harbor/SKILL.md

@@ -0,0 +1,80 @@
+---
+name: run-eval-harbor
+description: 'Run and diagnose existing Harbor evaluations against local production LobeHub or LobeHub Cloud. Use for eval infrastructure, target preflight, lh CLI injection, Harbor smoke or job runs, resume, and failure triage. Excludes authoring Harbor tasks and product acceptance.'
+---
+
+# Run Eval Harbor
+
+Run existing Harbor evaluations against either this checkout's isolated local
+production harness or a remote LobeHub target. Use `create-task` to author or
+grade tasks and `acceptance` for product acceptance.
+
+## Ask First
+
+Before preparing or running anything, obtain these independent choices:
+
+1. Target: `local` production server from this checkout, or `cloud`/remote.
+2. CLI: `checkout` build from `apps/cli`, or published `npm` release.
+3. Exact `LH_AGENT_ID`; the selected agent already owns its model.
+4. Eval repository path and whether the user wants a new job or a resume.
+5. Credentials: for cloud, require its CLI API key in the eval repository's
+   ignored `.env`. For local, ask whether the selected agent's provider
+   credential is already stored in LobeHub; if not, ask for the provider's real
+   environment variable name and secret before starting the server.
+6. For local, obtain explicit confirmation that port `3210` and every configured
+   eval infrastructure port are unreachable from the public internet and other
+   untrusted networks. Do not bootstrap the local stack without confirmation.
+
+Do not infer these choices. DeepSeek is only one provider example, not a
+required credential or model.
+
+## Guardrails
+
+- In local mode, run LobeHub on port `3210` with `bun run start`; never target a
+  dev server or port `3010`. Compose owns infrastructure; LobeHub stays on the
+  host.
+- In cloud mode, never start local infrastructure or rewrite server/gateway
+  addresses. Official Cloud should use the CLI's default addresses.
+- Create `docker-compose/eval/.env` from `.env.example` only when absent; never
+  overwrite an existing file.
+- The local Compose stack publishes host ports and uses fixed development
+  credentials, including the seeded CLI key and gateway service token. Never
+  run it on a host where those ports are reachable by an untrusted network.
+- Never infer `inbox`, choose a separate model, or override the agent with
+  `DEFAULT_AGENT_CONFIG`. Never invent, print, or commit secrets.
+- LobeHub uses localhost service URLs. Harbor containers use Docker-reachable
+  host URLs. Never interchange them.
+- Preflight is target-specific and read-only: local checks the local production
+  stack; cloud checks the remote server/gateways. It does not validate API keys,
+  agents, provider credentials, or model access.
+- Do not run a model-backed Harbor job without an explicit user request.
+- Before every requested real job or resume, run the shared model-backed smoke
+  for the chosen target and CLI mode. Stop if either preflight or smoke fails.
+
+## Run
+
+Read exactly one route after the answers above:
+
+- Local target: [references/local.md](references/local.md)
+- Cloud/remote target: [references/cloud.md](references/cloud.md)
+
+The CLI selection is orthogonal to the target. `checkout` injects the built
+`apps/cli`; `npm` installs the release package. Both routes run their preflight
+and then `scripts/run-smoke.sh <local|cloud> <checkout|npm> ...` before the
+external eval repository's own job command.
+
+## Diagnose
+
+- PostgreSQL, Redis, RustFS, QStash, or Compose state: local eval infrastructure.
+- Port `3210`, migrations, API-key auth, or `/api/version`: LobeHub.
+- Ports `8787`/`8788`, gateway health, or service tokens: gateway.
+- Docker-only connectivity: bridge address, published port, or host firewall.
+- CLI upload/install: `LH_CLI_SOURCE` or `apps/cli/dist`.
+- Reward/verifier behavior: the Harbor task; use `create-task` before changing it.
+
+## Harbor Reference
+
+For Harbor commands beyond these scripts, consult the official
+[Harbor Skills](https://github.com/harbor-framework/skills), especially its
+`harbor-cli` skill. This harness pins `harbor==0.23.0`; when guidance differs,
+the pinned CLI's `--help` is authoritative.

.agents/skills/run-eval-harbor/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: 'Run Eval Harbor'
+  short_description: 'Run Harbor evals against local or cloud LobeHub'
+  default_prompt: 'Use $run-eval-harbor to choose a LobeHub target and CLI source, preflight it, smoke-test the full path, and run an existing Harbor evaluation.'

.agents/skills/run-eval-harbor/references/cloud.md

@@ -0,0 +1,49 @@
+# Cloud Target
+
+Use this route for LobeHub Cloud or another remote production LobeHub server.
+Do not start local Compose, bootstrap a local user, launch `server.sh`, or
+replace any remote address supplied by the eval repository.
+
+## Prepare
+
+The external eval repository's ignored `.env` must contain:
+
+```env
+LH_AGENT_ID=<selected-agent-id>
+LOBEHUB_CLI_API_KEY=<cloud-cli-api-key>
+```
+
+For official LobeHub Cloud, leave `LH_SERVER_URL`, `LOBEHUB_SERVER`,
+`LH_GATEWAY_URL`, and `AGENT_GATEWAY_URL` unset so the CLI uses its official
+defaults. For a custom remote deployment, preserve its supplied URLs and require
+both `LH_GATEWAY_URL` and `AGENT_GATEWAY_URL`; if both `LH_SERVER_URL` and
+`LOBEHUB_SERVER` are present, they must match.
+
+The selected cloud agent already owns its model and provider configuration. Do
+not ask for or inject a local provider key.
+
+For CLI mode `checkout`, build this checkout before smoke:
+
+```bash
+pnpm --dir apps/cli build
+```
+
+CLI mode `npm` installs the published `@lobehub/cli` in the Harbor task.
+
+## Gate Before A Real Job
+
+Cloud preflight probes the selected server and available gateways from both the
+host and Docker. It is read-only and does not authenticate or call a model.
+
+```bash
+bash .agents/skills/run-eval-harbor/scripts/preflight.sh cloud /absolute/eval/repo
+bash .agents/skills/run-eval-harbor/scripts/run-smoke.sh cloud checkout /absolute/eval/repo
+# or: run-smoke.sh cloud npm /absolute/eval/repo
+```
+
+Both commands must pass before a real Harbor run or resume. The shared smoke is
+the proof that the API key, agent selection, CLI installation, device
+registration, gateways, and one actual model response work end to end.
+
+After smoke, use the external eval repository's own run/resume command. Never
+substitute a fresh job for a requested resume.

.agents/skills/run-eval-harbor/references/local.md

@@ -0,0 +1,92 @@
+# Local Target
+
+Use this route for a production LobeHub build from this checkout. Port `3010`
+is development-only; Harbor always targets the production server on `3210`.
+
+## Confirm Network Safety
+
+Before starting anything, ask the user to confirm that this machine is trusted
+and that LobeHub port `3210` plus all configured eval ports are unreachable from
+the public internet and other untrusted networks. The defaults are PostgreSQL
+`15433`, Redis `6380`, RustFS `9100`/`9101`, QStash `8080`/`8081`, and the
+gateways `8787`/`8788`. Stop if the user cannot confirm this.
+
+Compose port publishing is separate from container routing through the Docker
+bridge: an unqualified mapping such as `15433:5432` normally listens on every
+host interface. Compose services use their private bridge internally, the host
+LobeHub process uses published data-service ports, and Harbor containers use the
+host bridge address only for LobeHub and the gateways. The stack uses fixed
+development credentials, including its seeded CLI key and gateway service
+token, so it is not suitable for an internet-facing host.
+
+## Prepare
+
+Create `docker-compose/eval/.env` from `.env.example` only when it is absent.
+Set the user-selected `LH_AGENT_ID`. The agent already owns its model. Ask
+whether its provider credential is stored in LobeHub; when it is not, add the
+provider's actual environment variable to this ignored env before server
+startup. `DEEPSEEK_API_KEY` is one example, not a required or generic key.
+
+```bash
+bash .agents/skills/run-eval-harbor/scripts/bootstrap.sh
+bun --env-file=docker-compose/eval/.env run build
+bash .agents/skills/run-eval-harbor/scripts/server.sh
+```
+
+`server.sh` is long-running. Bootstrap starts PostgreSQL, Redis, RustFS, QStash,
+Device Gateway, and Agent Gateway, migrates the database, then seeds the eval
+user and CLI key. Keep `.records/env/eval-harbor-cli.env` private.
+
+For CLI mode `checkout`, also run:
+
+```bash
+pnpm --dir apps/cli build
+```
+
+CLI mode `npm` installs the published `@lobehub/cli` in each Harbor task and
+does not require a local CLI build.
+
+## Addresses
+
+LobeHub uses localhost service URLs. Harbor containers need host addresses:
+
+```env
+LH_SERVER_URL=http://172.17.0.1:3210
+LH_GATEWAY_URL=http://172.17.0.1:8787
+AGENT_GATEWAY_URL=http://172.17.0.1:8788
+```
+
+Resolve the bridge gateway instead of assuming `172.17.0.1`:
+
+```bash
+docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}'
+```
+
+## Gate Before A Real Job
+
+Run the local service preflight, then the model-backed smoke with the selected
+CLI mode. Both must pass before running or resuming a real Harbor job.
+
+```bash
+bash .agents/skills/run-eval-harbor/scripts/preflight.sh local
+bash .agents/skills/run-eval-harbor/scripts/preflight.sh local /absolute/eval/repo
+bash .agents/skills/run-eval-harbor/scripts/run-smoke.sh local checkout /absolute/eval/repo
+# or: run-smoke.sh local npm /absolute/eval/repo
+```
+
+The smoke requires `LH_AGENT_ID` and `LOBEHUB_CLI_API_KEY`, calls the selected
+agent, requires `hello world`, and exits. Preflight intentionally checks neither
+credential nor model access; that is smoke's job.
+
+Use the external eval repository's own run/resume command after smoke. Inspect
+failures from `<jobs-dir>/<job-id>/job.log`, then the failed trial's
+`exception.txt`, `agent/setup/`, `agent/command-*/`, and `verifier/` artifacts.
+
+## Stop
+
+```bash
+docker compose --env-file docker-compose/eval/.env \
+  -f docker-compose/eval/docker-compose.yml down
+```
+
+Do not add `-v` unless the user explicitly asks to discard eval data.

.agents/skills/run-eval-harbor/scripts/.gitignore

@@ -0,0 +1 @@
+__pycache__/

.agents/skills/run-eval-harbor/scripts/bootstrap.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)"
+ENV_FILE="${LOBEHUB_EVAL_ENV:-$REPO_ROOT/docker-compose/eval/.env}"
+ENV_EXAMPLE="$REPO_ROOT/docker-compose/eval/.env.example"
+COMPOSE_FILE="$REPO_ROOT/docker-compose/eval/docker-compose.yml"
+INIT_DEV_ENV="$REPO_ROOT/.agents/acceptance/scripts/init-dev-env.sh"
+JWKS_FILE="$REPO_ROOT/.records/env/agent-testing-jwks.json"
+CLI_ENV_FILE="$REPO_ROOT/.records/env/eval-harbor-cli.env"
+MODE="${1:-}"
+
+if [[ "$MODE" != "" && "$MODE" != "--infra-only" ]]; then
+  printf 'Usage: %s [--infra-only]\n' "$0" >&2
+  exit 2
+fi
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  cp "$ENV_EXAMPLE" "$ENV_FILE"
+  printf 'Created %s from .env.example\n' "$ENV_FILE"
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+
+# Reuse the repository acceptance bootstrap as the single source of truth for
+# the local signing key and seeded user contract.
+AGENT_TESTING_CLI_ENV_FILE="$CLI_ENV_FILE" \
+  bash "$INIT_DEV_ENV" --env-file "$ENV_FILE" env >/dev/null
+
+if [[ ! -s "$JWKS_FILE" ]]; then
+  printf 'init-dev-env.sh did not create %s\n' "$JWKS_FILE" >&2
+  exit 1
+fi
+
+export JWKS_KEY
+JWKS_KEY="$(tr -d '\n' < "$JWKS_FILE")"
+export EVAL_JWKS_PUBLIC_KEY
+EVAL_JWKS_PUBLIC_KEY="$(JWKS_FILE="$JWKS_FILE" node <<'NODE'
+const fs = require('node:fs');
+
+const privateJwks = JSON.parse(fs.readFileSync(process.env.JWKS_FILE, 'utf8'));
+const privateFields = new Set(['d', 'p', 'q', 'dp', 'dq', 'qi']);
+const keys = privateJwks.keys.map((key) =>
+  Object.fromEntries(Object.entries(key).filter(([name]) => !privateFields.has(name))),
+);
+process.stdout.write(JSON.stringify({ keys }));
+NODE
+)"
+
+docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d --wait
+
+if [[ "$MODE" == "--infra-only" ]]; then
+  exit 0
+fi
+
+cd "$REPO_ROOT"
+bun --env-file="$ENV_FILE" run db:migrate
+AGENT_TESTING_CLI_ENV_FILE="$CLI_ENV_FILE" \
+  bash "$INIT_DEV_ENV" --env-file "$ENV_FILE" seed-user >/dev/null
+
+printf 'Eval infrastructure, migrations, and baseline user are ready.\n'

.agents/skills/run-eval-harbor/scripts/lh/agent.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import shlex
+from pathlib import Path
+
+from harbor.agents.installed.base import BaseInstalledAgent
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+_HOST_DIR_PREFIX = "host-dir:"
+_DEV_CLI_DIR = "/opt/lh-dev"
+_DEV_CLI_RUNNER = f"{_DEV_CLI_DIR}/run-lh.sh"
+_CHECK_LH_PATH = "/installed-agent/check-lh.sh"
+_CONNECT_SCRIPT = "/tmp/lh-connect-supervised.sh"
+_LOGIN_READY = "/tmp/lh-login-ready"
+_DEVICE_READY = "/tmp/lh-device-ready"
+_SUPERVISOR_CONFIG = "/tmp/lh-supervisord.conf"
+_SUPERVISOR_SOCKET = "/tmp/lh-supervisor.sock"
+_TEMPLATE_DIR = Path(__file__).with_name("template")
+_TEMPLATES = Environment(
+    autoescape=False,
+    keep_trailing_newline=True,
+    loader=FileSystemLoader(_TEMPLATE_DIR),
+    undefined=StrictUndefined,
+)
+
+
+class LhInstalledAgent(BaseInstalledAgent):
+    def __init__(
+        self,
+        logs_dir: Path,
+        prompt_template_path: Path | str | None = None,
+        version: str | None = None,
+        extra_env: dict[str, str] | None = None,
+        agent_id: str | None = None,
+        server_url: str | None = None,
+        gateway_url: str | None = None,
+        cli_source: str | None = None,
+        *args,
+        **kwargs,
+    ):
+        self._agent_id = agent_id
+        self._server_url = server_url
+        self._gateway_url = gateway_url
+        self._cli_source_arg = cli_source
+        super().__init__(
+            logs_dir=logs_dir,
+            prompt_template_path=prompt_template_path,
+            version=version,
+            extra_env=extra_env,
+            *args,
+            **kwargs,
+        )
+
+    @staticmethod
+    def name() -> str:
+        return "lh"
+
+    def _value(self, direct: str | None, env_name: str, default: str = "") -> str:
+        return (direct or self._get_env(env_name) or default).strip()
+
+    @property
+    def _cli_source(self) -> str:
+        return self._value(self._cli_source_arg, "LH_CLI_SOURCE", "system")
+
+    def _render_template(self, name: str, **values: object) -> str:
+        return _TEMPLATES.get_template(name).render(**values)
+
+    def _host_cli_dir(self) -> Path:
+        if not self._cli_source.startswith(_HOST_DIR_PREFIX):
+            raise ValueError(f"Unsupported LH_CLI_SOURCE: {self._cli_source}")
+
+        path = Path(self._cli_source.removeprefix(_HOST_DIR_PREFIX)).expanduser()
+        if not path.is_absolute():
+            raise ValueError("LH_CLI_SOURCE host-dir path must be absolute")
+        for required in (path / "package.json", path / "dist" / "index.js"):
+            if not required.is_file():
+                raise FileNotFoundError(f"Missing local LH CLI build input: {required}")
+        return path
+
+    def _cli_command(self) -> str:
+        if self._cli_source == "system":
+            return "lh"
+        self._host_cli_dir()
+        return f"bash {_DEV_CLI_RUNNER}"
+
+    def _agent_target(self) -> tuple[str, str]:
+        agent_id = self._value(self._agent_id, "LH_AGENT_ID")
+        if agent_id:
+            return "--agent-id", agent_id
+        raise ValueError("LH_AGENT_ID is required")
+
+    async def install(self, environment: BaseEnvironment) -> None:
+        self.logs_dir.mkdir(parents=True, exist_ok=True)
+        await self.exec_as_root(
+            environment,
+            command=f"mkdir -p /installed-agent {shlex.quote(_DEV_CLI_DIR)}",
+        )
+
+        if self._cli_source != "system":
+            host_dir = self._host_cli_dir()
+            await environment.upload_file(host_dir / "package.json", f"{_DEV_CLI_DIR}/package.json")
+            await environment.upload_dir(host_dir / "dist", f"{_DEV_CLI_DIR}/dist")
+
+        install_script = self.logs_dir / "install-lh.sh"
+        install_script.write_text(
+            self._render_template(
+                "install-lh.sh.j2",
+                cli_package=shlex.quote("@lobehub/cli"),
+                node_version=shlex.quote(self._value(None, "LH_NODE_VERSION", "24")),
+                use_system_cli=self._cli_source == "system",
+            )
+        )
+        await environment.upload_file(install_script, "/installed-agent/install-lh.sh")
+        await self.exec_as_root(
+            environment,
+            command="chmod +x /installed-agent/install-lh.sh && /installed-agent/install-lh.sh",
+        )
+
+        check_script = self.logs_dir / "check-lh.sh"
+        check_script.write_text(self._render_template("check-lh.sh.j2"))
+        await environment.upload_file(check_script, _CHECK_LH_PATH)
+        await self.exec_as_root(
+            environment,
+            command=f"chmod +x {shlex.quote(_CHECK_LH_PATH)}",
+        )
+
+    def create_run_agent_commands(self, instruction: str) -> list[str]:
+        selector_flag, selector_value = self._agent_target()
+        server_url = self._value(self._server_url, "LH_SERVER_URL")
+        gateway_url = self._value(self._gateway_url, "LH_GATEWAY_URL")
+        cli = self._cli_command()
+
+        login = (
+            f"rm -f {_LOGIN_READY} {_DEVICE_READY}; "
+            f"({cli} whoami >/dev/null 2>&1 || {cli} login"
+        )
+        if server_url:
+            login += f" --server {shlex.quote(server_url)}"
+        login += f") && touch {_LOGIN_READY}"
+
+        connect = self._render_template(
+            "connect-lh.sh.j2",
+            cli_command=cli,
+            connect_script=_CONNECT_SCRIPT,
+            device_ready=_DEVICE_READY,
+            gateway_url=shlex.quote(gateway_url) if gateway_url else "",
+            login_ready=_LOGIN_READY,
+            supervisor_config=_SUPERVISOR_CONFIG,
+            supervisor_socket=_SUPERVISOR_SOCKET,
+        )
+        ready = (
+            f"test -f {_LOGIN_READY} || exit 1; "
+            f"{_CHECK_LH_PATH} -- {cli} && touch {_DEVICE_READY}"
+        )
+        run = self._render_template(
+            "run-agent.sh.j2",
+            cli_command=cli,
+            device_ready=_DEVICE_READY,
+            instruction=shlex.quote(instruction),
+            selector_flag=selector_flag,
+            selector_value=shlex.quote(selector_value),
+            supervisor_config=_SUPERVISOR_CONFIG,
+        )
+        return [login, connect, ready, run]
+
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        del context
+        for command in self.create_run_agent_commands(self.render_instruction(instruction)):
+            await self.exec_as_agent(environment, command=command)

.agents/skills/run-eval-harbor/scripts/lh/template/check-lh.sh.j2

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MAX_ATTEMPTS="${LH_LOCAL_DEVICE_READY_ATTEMPTS:-30}"
+STATUS_PATH="$HOME/.lobehub/daemon.status.json"
+
+[[ "${1:-}" == "--" ]] && shift
+[[ "$#" -gt 0 ]] || { echo "Usage: check-lh.sh -- <lh-command>" >&2; exit 2; }
+CLI_COMMAND=("$@")
+
+for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do
+  if device_id="$(node - "$STATUS_PATH" <<'NODE'
+const fs = require('node:fs');
+try {
+  const status = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
+  if (status.connectionStatus !== 'connected' || !status.deviceId) process.exit(1);
+  process.stdout.write(status.deviceId);
+} catch {
+  process.exit(1);
+}
+NODE
+  )" && [[ -n "$device_id" ]]; then
+    if "${CLI_COMMAND[@]}" device list --json deviceId,online 2>/dev/null \
+      | LOCAL_DEVICE_ID="$device_id" node -e '
+let input = "";
+process.stdin.setEncoding("utf8");
+process.stdin.on("data", (chunk) => { input += chunk; });
+process.stdin.on("end", () => {
+  try {
+    const devices = JSON.parse(input);
+    process.exit(Array.isArray(devices) && devices.some(
+      (device) => device.deviceId === process.env.LOCAL_DEVICE_ID && device.online === true,
+    ) ? 0 : 1);
+  } catch { process.exit(1); }
+});'; then
+      echo "Local device ${device_id} is online in the gateway registry."
+      exit 0
+    fi
+  fi
+  sleep 1
+done
+
+echo "lh local device never became connected and online." >&2
+"${CLI_COMMAND[@]}" connect logs --lines 200 >&2 || true
+exit 1

.agents/skills/run-eval-harbor/scripts/lh/template/connect-lh.sh.j2

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+test -f {{ login_ready }} || {
+  echo "lh login did not complete" >&2
+  exit 1
+}
+rm -f {{ device_ready }}
+mkdir -p "$HOME/.lobehub" /logs/agent
+rm -f "$HOME/.lobehub/daemon.log"
+ln -s /logs/agent/daemon.log "$HOME/.lobehub/daemon.log"
+
+cat > {{ connect_script }} <<'LH_CONNECT'
+#!/usr/bin/env bash
+set -euo pipefail
+mkdir -p "$HOME/.lobehub" /logs/agent
+printf "%s\n" "$$" > "$HOME/.lobehub/daemon.pid"
+exec {{ cli_command }} connect --daemon-child{% if gateway_url %} --gateway {{ gateway_url }}{% endif %}
+LH_CONNECT
+chmod +x {{ connect_script }}
+
+cat > {{ supervisor_config }} <<'LH_SUPERVISOR'
+{% include "supervisord.conf.j2" -%}
+LH_SUPERVISOR
+supervisord -c {{ supervisor_config }}

.agents/skills/run-eval-harbor/scripts/lh/template/install-lh.sh.j2

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CLI_PACKAGE={{ cli_package }}
+NODE_VERSION={{ node_version }}
+REQUIRED_NODE_MAJOR="${NODE_VERSION%%.*}"
+REQUIRED_NODE_MAJOR="${REQUIRED_NODE_MAJOR#v}"
+NVM_DIR="${HOME}/.nvm"
+
+have_compatible_node() {
+  command -v node >/dev/null 2>&1 || return 1
+  current_major="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || true)"
+  [[ -n "$current_major" && "$current_major" -ge "$REQUIRED_NODE_MAJOR" ]]
+}
+
+install_node() {
+  if [[ ! -s "$NVM_DIR/nvm.sh" ]]; then
+    command -v curl >/dev/null 2>&1 || {
+      apt-get update
+      DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates
+    }
+    curl -4 -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash
+  fi
+  # shellcheck disable=SC1091
+  . "$NVM_DIR/nvm.sh"
+  nvm install "$NODE_VERSION"
+  nvm use "$NODE_VERSION" >/dev/null
+}
+
+if ! have_compatible_node; then
+  install_node
+fi
+
+if ! command -v supervisord >/dev/null 2>&1; then
+  apt-get update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y supervisor
+fi
+
+for bin in node npm; do
+  bin_path="$(command -v "$bin")"
+  [[ "$bin_path" == "/usr/local/bin/$bin" ]] || ln -sf "$bin_path" "/usr/local/bin/$bin"
+done
+
+{% if use_system_cli %}
+npm install -g "$CLI_PACKAGE"
+lh_path="$(command -v lh)"
+[[ "$lh_path" == "/usr/local/bin/lh" ]] || ln -sf "$lh_path" /usr/local/bin/lh
+/usr/local/bin/lh --version
+{% else %}
+test -f /opt/lh-dev/package.json
+test -f /opt/lh-dev/dist/index.js
+cat >/opt/lh-dev/run-lh.sh <<'LH_RUNNER'
+#!/usr/bin/env bash
+set -euo pipefail
+exec node /opt/lh-dev/dist/index.js "$@"
+LH_RUNNER
+chmod +x /opt/lh-dev/run-lh.sh
+/opt/lh-dev/run-lh.sh --version
+{% endif %}

.agents/skills/run-eval-harbor/scripts/lh/template/run-agent.sh.j2

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+test -f {{ device_ready }} || {
+  echo "lh local device is not ready" >&2
+  exit 1
+}
+
+agent_log=/logs/agent/agent-run.log
+rm -f "$agent_log"
+
+cleanup() {
+  supervisorctl -c {{ supervisor_config }} shutdown >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+set +e
+{{ cli_command }} agent run {{ selector_flag }} {{ selector_value }} \
+  --prompt {{ instruction }} \
+  --device local \
+  2>&1 | tee -a "$agent_log"
+rc="${PIPESTATUS[0]}"
+set -e
+exit "$rc"

.agents/skills/run-eval-harbor/scripts/lh/template/supervisord.conf.j2

@@ -0,0 +1,27 @@
+[supervisord]
+nodaemon=false
+pidfile=/tmp/lh-supervisord.pid
+logfile=/logs/agent/supervisord.log
+childlogdir=/logs/agent
+
+[unix_http_server]
+file={{ supervisor_socket }}
+
+[supervisorctl]
+serverurl=unix://{{ supervisor_socket }}
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:lh-connect]
+command={{ connect_script }}
+autostart=true
+autorestart=unexpected
+exitcodes=0
+startsecs=2
+startretries=999
+stopsignal=TERM
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/logs/agent/daemon.log
+redirect_stderr=true

.agents/skills/run-eval-harbor/scripts/preflight.sh

@@ -0,0 +1,370 @@
+#!/usr/bin/env bash
+
+set -u -o pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="${LOBEHUB_REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null)}"
+ENV_FILE="${LOBEHUB_EVAL_ENV:-${REPO_ROOT}/docker-compose/eval/.env}"
+TARGET="${1:-}"
+HARBOR_REPO="${2:-}"
+COMPOSE_FILE="${REPO_ROOT}/docker-compose/eval/docker-compose.yml"
+CURL_IMAGE="curlimages/curl:8.17.0"
+JWKS_FILE="$REPO_ROOT/.records/env/agent-testing-jwks.json"
+
+if [[ "$TARGET" == '-h' || "$TARGET" == '--help' ]]; then
+  printf 'Usage: %s <local|cloud> [absolute-eval-repository]\n' "$0"
+  exit 0
+fi
+if [[ "$TARGET" != 'local' && "$TARGET" != 'cloud' ]]; then
+  printf 'Usage: %s <local|cloud> [absolute-eval-repository]\n' "$0" >&2
+  exit 2
+fi
+if [[ -n "$HARBOR_REPO" && ! -d "$HARBOR_REPO" ]]; then
+  printf 'FAIL target eval repository does not exist: %s\n' "$HARBOR_REPO" >&2
+  exit 2
+fi
+if [[ -n "$HARBOR_REPO" ]]; then
+  HARBOR_REPO="$(cd -- "$HARBOR_REPO" && pwd)"
+  HARBOR_ENV="$HARBOR_REPO/.env"
+else
+  HARBOR_ENV=""
+fi
+if [[ "$TARGET" == 'cloud' && -z "$HARBOR_ENV" ]]; then
+  printf 'FAIL cloud preflight requires an eval repository with a .env file\n' >&2
+  exit 2
+fi
+
+passes=0
+failures=0
+
+pass() {
+  passes=$((passes + 1))
+  printf 'PASS %s\n' "$*"
+}
+
+fail() {
+  failures=$((failures + 1))
+  printf 'FAIL %s\n' "$*" >&2
+}
+
+section() {
+  printf '\n%s\n' "$*"
+}
+
+need_cmd() {
+  if command -v "$1" >/dev/null 2>&1; then
+    pass "command available: $1"
+  else
+    fail "missing command: $1"
+  fi
+}
+
+read_dotenv() {
+  local key="$1"
+  local file="${2:-$ENV_FILE}"
+  [[ -f "$file" ]] || return 0
+  python3 - "$key" "$file" <<'PY'
+from pathlib import Path
+import sys
+
+key, path = sys.argv[1], Path(sys.argv[2])
+for raw in path.read_text(errors="ignore").splitlines():
+    line = raw.strip()
+    if not line or line.startswith("#") or "=" not in line:
+        continue
+    if line.startswith("export "):
+        line = line[len("export "):].strip()
+    name, value = line.split("=", 1)
+    if name.strip() == key:
+        print(value.strip().strip("'\""))
+        break
+PY
+}
+
+require_env() {
+  local key="$1"
+  local file="${2:-$ENV_FILE}"
+  if [[ -n "$(read_dotenv "$key" "$file")" ]]; then
+    pass "$key is configured"
+  else
+    fail "$key is missing in $file"
+  fi
+}
+
+expect_env() {
+  local key="$1"
+  local expected="$2"
+  local actual
+  actual="$(read_dotenv "$key")"
+  if [[ "$actual" == "$expected" ]]; then
+    pass "$key has the required value"
+  else
+    fail "$key must be $expected"
+  fi
+}
+
+http_probe() {
+  local label="$1"
+  local url="$2"
+  local header="${3:-}"
+  local args=(-L -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10)
+  [[ -n "$header" ]] && args+=(-H "$header")
+  local code
+  code="$(curl "${args[@]}" "$url" 2>/dev/null || true)"
+  if [[ "$code" =~ ^[23][0-9][0-9]$ ]]; then
+    pass "$label reachable ($code)"
+  else
+    fail "$label unreachable or unhealthy ($url, HTTP ${code:-000})"
+  fi
+}
+
+container_probe() {
+  local label="$1"
+  local url="$2"
+  local output
+  output="$(docker run --rm "$CURL_IMAGE" -sS -o /dev/null -w '%{http_code}' \
+    --connect-timeout 3 --max-time 10 "$url" 2>/dev/null || true)"
+  if [[ "$output" =~ ^[23][0-9][0-9]$ ]]; then
+    pass "$label reachable from Docker ($output)"
+  else
+    fail "$label unreachable from Docker ($url, HTTP ${output:-000})"
+  fi
+}
+
+compose_id() {
+  local service="$1"
+  docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" ps -a -q "$service" 2>/dev/null | head -n 1
+}
+
+if [[ "$TARGET" == 'cloud' ]]; then
+  section 'Cloud target configuration'
+  [[ -f "$HARBOR_ENV" ]] && pass "eval env: $HARBOR_ENV" || fail "missing eval env: $HARBOR_ENV"
+  for command in docker curl python3; do
+    need_cmd "$command"
+  done
+  docker info >/dev/null 2>&1 && pass 'Docker daemon is available' || fail 'Docker daemon is unavailable'
+  cloud_lh_server="$(read_dotenv LH_SERVER_URL "$HARBOR_ENV")"
+  cloud_cli_server="$(read_dotenv LOBEHUB_SERVER "$HARBOR_ENV")"
+  if [[ -n "$cloud_lh_server" && -n "$cloud_cli_server" && "$cloud_lh_server" != "$cloud_cli_server" ]]; then
+    fail 'LH_SERVER_URL and LOBEHUB_SERVER must match when both are set'
+  fi
+  cloud_server="${cloud_lh_server:-${cloud_cli_server:-https://app.lobehub.com}}"
+  cloud_device_gateway="$(read_dotenv LH_GATEWAY_URL "$HARBOR_ENV")"
+  cloud_agent_gateway="$(read_dotenv AGENT_GATEWAY_URL "$HARBOR_ENV")"
+
+  if [[ "$cloud_server" == 'https://app.lobehub.com' ]]; then
+    cloud_device_gateway="${cloud_device_gateway:-https://device-gateway.lobehub.com}"
+    cloud_agent_gateway="${cloud_agent_gateway:-https://agent-gateway.lobehub.com}"
+  else
+    [[ -n "$cloud_device_gateway" ]] || fail 'custom remote server requires LH_GATEWAY_URL'
+    [[ -n "$cloud_agent_gateway" ]] || fail 'custom remote server requires AGENT_GATEWAY_URL'
+  fi
+
+  section 'Cloud service health'
+  http_probe 'LobeHub cloud server' "${cloud_server%/}/api/version"
+  container_probe 'LobeHub cloud server' "${cloud_server%/}/api/version"
+  if [[ -n "$cloud_device_gateway" ]]; then
+    http_probe 'Device Gateway' "${cloud_device_gateway%/}/health"
+    container_probe 'Device Gateway' "${cloud_device_gateway%/}/health"
+  fi
+  if [[ -n "$cloud_agent_gateway" ]]; then
+    http_probe 'Agent Gateway' "${cloud_agent_gateway%/}/health"
+    container_probe 'Agent Gateway' "${cloud_agent_gateway%/}/health"
+  fi
+
+  section 'Summary'
+  printf 'Passes: %d\nFailures: %d\n' "$passes" "$failures"
+  if ((failures > 0)); then
+    exit 1
+  fi
+  exit 0
+fi
+
+section 'Files and commands'
+[[ -n "$REPO_ROOT" && -f "$REPO_ROOT/package.json" ]] && pass "LobeHub repo: $REPO_ROOT" || fail 'cannot resolve the LobeHub repository root'
+[[ -f "$ENV_FILE" ]] && pass "eval env: $ENV_FILE" || fail "missing $ENV_FILE; copy .env.example first"
+[[ -f "$COMPOSE_FILE" ]] && pass "Compose file: $COMPOSE_FILE" || fail "missing Compose file: $COMPOSE_FILE"
+
+for command in docker curl python3 node ss; do
+  need_cmd "$command"
+done
+
+if ! docker info >/dev/null 2>&1; then
+  fail 'Docker daemon is unavailable'
+fi
+
+if docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" config --quiet >/dev/null 2>&1; then
+  pass 'Compose configuration is valid'
+else
+  fail 'Compose configuration is invalid'
+fi
+
+section 'Production LobeHub contract'
+expect_env APP_URL http://localhost:3210
+expect_env INTERNAL_APP_URL http://localhost:3210
+expect_env EVAL_LOBE_API_BASE_URL http://host.docker.internal:3210
+expect_env DATABASE_DRIVER node
+expect_env AGENT_RUNTIME_BASE_URL http://localhost:3210
+expect_env AGENT_RUNTIME_MODE queue
+expect_env DEVICE_GATEWAY_URL http://localhost:8787
+expect_env AGENT_GATEWAY_URL http://localhost:8788
+expect_env NEXT_PUBLIC_SERVICE_MODE server
+expect_env TB_GRAPH_AGENT 1
+expect_env ENABLE_AGENT_FILE_TRACING 1
+expect_env ENABLE_AGENT_GATEWAY 1
+
+for key in \
+  DATABASE_URL REDIS_URL S3_ENDPOINT S3_BUCKET QSTASH_URL QSTASH_TOKEN \
+  QSTASH_CURRENT_SIGNING_KEY QSTASH_NEXT_SIGNING_KEY DEVICE_GATEWAY_URL \
+  DEVICE_GATEWAY_SERVICE_TOKEN AGENT_GATEWAY_URL AGENT_GATEWAY_SERVICE_TOKEN \
+  AGENT_RUNTIME_BASE_URL KEY_VAULTS_SECRET AUTH_SECRET LH_SERVER_URL \
+  LH_GATEWAY_URL HARBOR_AGENT_GATEWAY_URL; do
+  require_env "$key"
+done
+
+[[ -s "$JWKS_FILE" ]] && pass 'eval JWKS private key exists' || fail "missing $JWKS_FILE; run the eval bootstrap script"
+
+gateway_token="$(read_dotenv EVAL_GATEWAY_SERVICE_TOKEN)"
+device_token="$(read_dotenv DEVICE_GATEWAY_SERVICE_TOKEN)"
+agent_token="$(read_dotenv AGENT_GATEWAY_SERVICE_TOKEN)"
+if [[ -n "$gateway_token" && "$gateway_token" == "$device_token" && "$gateway_token" == "$agent_token" ]]; then
+  pass 'unified gateway service tokens match'
+else
+  fail 'EVAL_GATEWAY_SERVICE_TOKEN must match both LobeHub gateway tokens'
+fi
+
+section 'Compose services'
+for service in postgresql redis rustfs qstash gateway; do
+  id="$(compose_id "$service")"
+  if [[ -z "$id" ]]; then
+    fail "$service container does not exist"
+    continue
+  fi
+  status="$(docker inspect -f '{{.State.Status}}' "$id" 2>/dev/null || true)"
+  health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id" 2>/dev/null || true)"
+  if [[ "$status" == 'running' && "$health" != 'unhealthy' ]]; then
+    pass "$service running (health: $health)"
+  else
+    fail "$service not ready (status: ${status:-unknown}, health: ${health:-unknown})"
+  fi
+done
+
+init_id="$(compose_id rustfs-init)"
+init_exit="$(docker inspect -f '{{.State.ExitCode}}' "$init_id" 2>/dev/null || true)"
+[[ -n "$init_id" && "$init_exit" == '0' ]] && pass 'rustfs-init completed' || fail "rustfs-init failed or missing (exit: ${init_exit:-unknown})"
+
+gateway_id="$(compose_id gateway)"
+gateway_jwks="$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$gateway_id" 2>/dev/null | sed -n 's/^JWKS_PUBLIC_KEY=//p' | head -n 1)"
+expected_gateway_jwks="$(JWKS_FILE="$JWKS_FILE" node <<'NODE' 2>/dev/null || true
+const fs = require('node:fs');
+const privateJwks = JSON.parse(fs.readFileSync(process.env.JWKS_FILE, 'utf8'));
+const privateFields = new Set(['d', 'p', 'q', 'dp', 'dq', 'qi']);
+const keys = privateJwks.keys.map((key) =>
+  Object.fromEntries(Object.entries(key).filter(([name]) => !privateFields.has(name))),
+);
+process.stdout.write(JSON.stringify({ keys }));
+NODE
+)"
+if [[ -n "$gateway_jwks" && "$gateway_jwks" == "$expected_gateway_jwks" ]]; then
+  pass 'Gateway JWKS matches the production server signing key'
+else
+  fail 'Gateway JWKS is missing or does not match the production server signing key'
+fi
+
+postgres_id="$(compose_id postgresql)"
+postgres_db="$(read_dotenv EVAL_POSTGRES_DB)"
+postgres_db="${postgres_db:-lobechat}"
+extensions="$(docker exec "$postgres_id" psql -U postgres -d "$postgres_db" -Atqc \
+  "select extname from pg_extension where extname in ('pg_search','vector') order by extname" 2>/dev/null | tr '\n' ' ' || true)"
+if [[ "$extensions" == *pg_search* && "$extensions" == *vector* ]]; then
+  pass "PostgreSQL extensions installed: $extensions"
+else
+  fail 'PostgreSQL migrations have not installed pg_search and vector'
+fi
+
+redis_id="$(compose_id redis)"
+if [[ -n "$redis_id" ]] && docker exec "$redis_id" redis-cli ping 2>/dev/null | grep -qx PONG; then
+  pass 'Redis responds to PING'
+else
+  fail 'Redis does not respond to PING'
+fi
+
+section 'Host endpoints'
+rustfs_port="$(read_dotenv EVAL_RUSTFS_PORT)"
+device_port="$(read_dotenv EVAL_DEVICE_GATEWAY_PORT)"
+agent_port="$(read_dotenv EVAL_AGENT_GATEWAY_PORT)"
+qstash_url="$(read_dotenv QSTASH_URL)"
+qstash_token="$(read_dotenv QSTASH_TOKEN)"
+http_probe 'RustFS' "http://localhost:${rustfs_port:-9100}/health"
+http_probe 'Device Gateway' "http://localhost:${device_port:-8787}/health"
+http_probe 'Agent Gateway' "http://localhost:${agent_port:-8788}/health"
+http_probe 'QStash API' "${qstash_url%/}/v2/logs" "Authorization: Bearer $qstash_token"
+http_probe 'LobeHub production server' 'http://localhost:3210/api/version'
+
+if [[ -f "$REPO_ROOT/.next/BUILD_ID" ]]; then
+  pass 'LobeHub production build exists'
+else
+  fail 'missing .next/BUILD_ID; build before starting the production server'
+fi
+
+server_pid="$(ss -ltnp 'sport = :3210' 2>/dev/null | sed -n 's/.*pid=\([0-9][0-9]*\).*/\1/p' | head -n 1)"
+if [[ -n "$server_pid" ]]; then
+  pass "port 3210 listener found (pid: $server_pid)"
+  server_cwd="$(readlink -f "/proc/$server_pid/cwd" 2>/dev/null || true)"
+  if [[ "$server_cwd" == "$REPO_ROOT" ]]; then
+    pass 'port 3210 process runs from this checkout'
+  else
+    fail "port 3210 process cwd must be $REPO_ROOT"
+  fi
+
+  server_node_env="$(tr '\0' '\n' <"/proc/$server_pid/environ" 2>/dev/null | sed -n 's/^NODE_ENV=//p' | head -n 1 || true)"
+  if [[ "$server_node_env" == 'production' ]]; then
+    pass 'port 3210 process uses NODE_ENV=production'
+  else
+    fail 'port 3210 is not a verified production LobeHub process'
+  fi
+
+  if tr '\0' '\n' <"/proc/$server_pid/environ" 2>/dev/null | grep -q '^JWKS_KEY=.'; then
+    pass 'production server has JWKS_KEY'
+  else
+    fail 'production server is missing JWKS_KEY'
+  fi
+else
+  fail 'cannot identify the process listening on port 3210'
+fi
+
+lh_server_url="$(read_dotenv LH_SERVER_URL)"
+lh_gateway_url="$(read_dotenv LH_GATEWAY_URL)"
+harbor_agent_gateway_url="$(read_dotenv HARBOR_AGENT_GATEWAY_URL)"
+
+section 'Harbor configuration'
+if [[ -n "$HARBOR_ENV" ]]; then
+  pass "external eval repo: $HARBOR_REPO"
+  [[ -f "$HARBOR_ENV" ]] && pass "eval repo env: $HARBOR_ENV" || fail 'target eval repository is missing .env'
+  for key in LH_SERVER_URL LH_GATEWAY_URL AGENT_GATEWAY_URL; do
+    require_env "$key" "$HARBOR_ENV"
+  done
+  harbor_server="$(read_dotenv LH_SERVER_URL "$HARBOR_ENV")"
+  harbor_device_gateway="$(read_dotenv LH_GATEWAY_URL "$HARBOR_ENV")"
+  harbor_agent_gateway="$(read_dotenv AGENT_GATEWAY_URL "$HARBOR_ENV")"
+  [[ -z "$(read_dotenv LH_AGENT_RUN_SSE "$HARBOR_ENV")" ]] && pass 'eval repo uses gateway mode' || fail 'unset LH_AGENT_RUN_SSE when AGENT_GATEWAY_URL is configured'
+else
+  pass 'using repository-level Harbor smoke configuration'
+  harbor_server="$lh_server_url"
+  harbor_device_gateway="$lh_gateway_url"
+  harbor_agent_gateway="$harbor_agent_gateway_url"
+fi
+
+[[ "$harbor_server" == "$lh_server_url" ]] && pass 'eval repo LH_SERVER_URL matches harness' || fail 'eval repo LH_SERVER_URL differs from harness'
+[[ "$harbor_device_gateway" == "$lh_gateway_url" ]] && pass 'eval repo LH_GATEWAY_URL matches harness' || fail 'eval repo LH_GATEWAY_URL differs from harness'
+[[ "$harbor_agent_gateway" == "$harbor_agent_gateway_url" ]] && pass 'eval repo AGENT_GATEWAY_URL matches harness' || fail 'eval repo AGENT_GATEWAY_URL differs from harness'
+
+section 'Harbor container reachability'
+[[ "$lh_server_url" == *:3210 ]] && pass 'LH_SERVER_URL targets port 3210' || fail 'LH_SERVER_URL must target production port 3210'
+container_probe 'LobeHub' "${lh_server_url%/}/api/version"
+container_probe 'Device Gateway' "${lh_gateway_url%/}/health"
+container_probe 'Agent Gateway' "${harbor_agent_gateway_url%/}/health"
+
+section 'Summary'
+printf 'Passes: %d\nFailures: %d\n' "$passes" "$failures"
+((failures == 0))

.agents/skills/run-eval-harbor/scripts/run-smoke.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)"
+ENV_FILE="${LOBEHUB_EVAL_ENV:-$REPO_ROOT/docker-compose/eval/.env}"
+CLI_ENV_FILE="$REPO_ROOT/.records/env/eval-harbor-cli.env"
+TARGET="${1:-}"
+CLI_MODE="${2:-}"
+EVAL_REPO="${3:-}"
+SMOKE_DIR="$SCRIPT_DIR/smoke"
+JOBS_DIR="$REPO_ROOT/.records/harbor/jobs"
+JOB_NAME="smoke-$(date +%Y%m%d-%H%M%S)-$$"
+JOB_DIR="$JOBS_DIR/$JOB_NAME"
+
+usage() {
+  printf 'Usage: %s <local|cloud> <checkout|npm> [absolute-eval-repository]\n' "$0"
+}
+
+if [[ "$TARGET" == '-h' || "$TARGET" == '--help' ]]; then
+  usage
+  exit 0
+fi
+[[ "$TARGET" == 'local' || "$TARGET" == 'cloud' ]] || { usage >&2; exit 2; }
+[[ "$CLI_MODE" == 'checkout' || "$CLI_MODE" == 'npm' ]] || { usage >&2; exit 2; }
+if [[ -n "$EVAL_REPO" ]]; then
+  [[ -d "$EVAL_REPO" ]] || { printf 'Eval repository does not exist: %s\n' "$EVAL_REPO" >&2; exit 2; }
+  EVAL_REPO="$(cd -- "$EVAL_REPO" && pwd)"
+  EVAL_ENV="$EVAL_REPO/.env"
+  [[ -f "$EVAL_ENV" ]] || { printf 'Missing eval environment: %s\n' "$EVAL_ENV" >&2; exit 1; }
+else
+  EVAL_ENV=""
+fi
+
+set -a
+if [[ "$TARGET" == 'local' ]]; then
+  [[ -f "$ENV_FILE" ]] || { printf 'Missing %s; run the eval bootstrap script first.\n' "$ENV_FILE" >&2; exit 1; }
+  [[ -f "$CLI_ENV_FILE" ]] || { printf 'Missing %s; run the eval bootstrap script first.\n' "$CLI_ENV_FILE" >&2; exit 1; }
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+  # shellcheck disable=SC1090
+  source "$CLI_ENV_FILE"
+  if [[ -n "$EVAL_ENV" ]]; then
+    # shellcheck disable=SC1090
+    source "$EVAL_ENV"
+  fi
+  export LOBEHUB_CLI_API_KEY="${LOBEHUB_CLI_API_KEY:-${LOBE_API_KEY:-}}"
+  export LOBEHUB_SERVER="${LH_SERVER_URL:-}"
+  export AGENT_GATEWAY_URL="${HARBOR_AGENT_GATEWAY_URL:-}"
+else
+  [[ -n "$EVAL_ENV" ]] || { printf 'Cloud smoke requires an eval repository with a .env file.\n' >&2; exit 2; }
+  # shellcheck disable=SC1090
+  source "$EVAL_ENV"
+fi
+set +a
+
+[[ -n "${LH_AGENT_ID:-}" ]] || { printf 'LH_AGENT_ID is required for smoke.\n' >&2; exit 1; }
+[[ -n "${LOBEHUB_CLI_API_KEY:-}" ]] || { printf 'LOBEHUB_CLI_API_KEY is required for smoke.\n' >&2; exit 1; }
+
+if [[ "$CLI_MODE" == 'checkout' ]]; then
+  CLI_DIR="$REPO_ROOT/apps/cli"
+  [[ -f "$CLI_DIR/package.json" && -f "$CLI_DIR/dist/index.js" ]] || {
+    printf 'Missing local CLI build; run pnpm --dir apps/cli build.\n' >&2
+    exit 1
+  }
+  export LH_CLI_SOURCE="host-dir:$CLI_DIR"
+else
+  export LH_CLI_SOURCE=system
+fi
+
+preflight_args=("$TARGET")
+[[ -n "$EVAL_REPO" ]] && preflight_args+=("$EVAL_REPO")
+bash "$SCRIPT_DIR/preflight.sh" "${preflight_args[@]}"
+mkdir -p "$JOBS_DIR"
+
+cd "$REPO_ROOT"
+export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}"
+export PYTHONDONTWRITEBYTECODE=1
+harbor_args=(
+  run
+  --path "$SMOKE_DIR"
+  --jobs-dir "$JOBS_DIR"
+  --job-name "$JOB_NAME"
+  --n-concurrent 1
+  --no-delete
+  --disable-verification
+  --yes
+  --agent lh.agent:LhInstalledAgent
+  --agent-env 'LH_AGENT_ID=${LH_AGENT_ID}'
+  --agent-env 'LOBEHUB_CLI_API_KEY=${LOBEHUB_CLI_API_KEY}'
+  --agent-env 'LH_CLI_SOURCE=${LH_CLI_SOURCE}'
+)
+for key in LH_SERVER_URL LH_GATEWAY_URL AGENT_GATEWAY_URL LOBEHUB_SERVER; do
+  [[ -n "${!key:-}" ]] && harbor_args+=(--agent-env "$key=\${$key}")
+done
+uv run --with 'harbor==0.23.0' harbor "${harbor_args[@]}"
+
+RESULT_FILE="$JOB_DIR/result.json" node <<'NODE'
+const fs = require('node:fs');
+const pathModule = require('node:path');
+
+const path = process.env.RESULT_FILE;
+if (!fs.existsSync(path)) throw new Error(`Harbor did not write ${path}`);
+const result = JSON.parse(fs.readFileSync(path, 'utf8'));
+const stats = result.stats || {};
+if (stats.n_completed_trials !== 1 || stats.n_errored_trials !== 0) {
+  throw new Error(
+    `Harbor smoke failed: completed=${stats.n_completed_trials ?? 0}, errored=${stats.n_errored_trials ?? 0}; inspect ${path}`,
+  );
+}
+
+const jobDir = pathModule.dirname(path);
+const agentLogs = fs
+  .readdirSync(jobDir, { withFileTypes: true })
+  .filter((entry) => entry.isDirectory())
+  .map((entry) => pathModule.join(jobDir, entry.name, 'agent', 'agent-run.log'))
+  .filter((candidate) => fs.existsSync(candidate));
+if (agentLogs.length !== 1) {
+  throw new Error(`Expected one Harbor smoke agent log, found ${agentLogs.length}; inspect ${jobDir}`);
+}
+const agentLog = fs.readFileSync(agentLogs[0], 'utf8');
+if (!/^hello world\r?$/m.test(agentLog) || !/Agent finished/.test(agentLog)) {
+  throw new Error(`Harbor smoke did not complete the hello-world agent run; inspect ${agentLogs[0]}`);
+}
+console.log(`Harbor smoke passed: ${path}`);
+NODE

.agents/skills/run-eval-harbor/scripts/server.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)"
+ENV_FILE="${LOBEHUB_EVAL_ENV:-$REPO_ROOT/docker-compose/eval/.env}"
+JWKS_FILE="$REPO_ROOT/.records/env/agent-testing-jwks.json"
+
+[[ -f "$ENV_FILE" ]] || { printf 'Missing %s; run the eval bootstrap script first.\n' "$ENV_FILE" >&2; exit 1; }
+[[ -s "$JWKS_FILE" ]] || { printf 'Missing %s; run the eval bootstrap script first.\n' "$JWKS_FILE" >&2; exit 1; }
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+export JWKS_KEY
+JWKS_KEY="$(tr -d '\n' < "$JWKS_FILE")"
+export NODE_ENV=production
+
+cd "$REPO_ROOT"
+exec bun run start

.agents/skills/run-eval-harbor/scripts/smoke/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:24.04
+
+WORKDIR /app

.agents/skills/run-eval-harbor/scripts/smoke/instruction.md

@@ -0,0 +1 @@
+Reply with exactly `hello world`, then exit.

.agents/skills/run-eval-harbor/scripts/smoke/task.toml

@@ -0,0 +1,13 @@
+version = "1.0"
+
+[metadata]
+name = "LobeHub Harbor end-to-end smoke"
+
+[agent]
+timeout_sec = 300.0
+
+[environment]
+build_timeout_sec = 300.0
+cpus = 1
+memory_mb = 1024
+storage_mb = 1024

docker-compose/eval/.env.example

@@ -0,0 +1,66 @@
+# Compose project and host ports. Defaults avoid the regular dev stack's data services.
+EVAL_COMPOSE_PROJECT_NAME=lobehub-eval
+EVAL_POSTGRES_PORT=15433
+EVAL_REDIS_PORT=6380
+EVAL_RUSTFS_PORT=9100
+EVAL_RUSTFS_ADMIN_PORT=9101
+EVAL_QSTASH_PORT=8080
+EVAL_QSTASH_LOG_PORT=8081
+EVAL_DEVICE_GATEWAY_PORT=8787
+EVAL_AGENT_GATEWAY_PORT=8788
+
+# Local infrastructure credentials. The two LobeHub gateway token variables must
+# use the same value because the unified gateway exposes one SERVICE_TOKEN.
+EVAL_POSTGRES_DB=lobechat
+EVAL_POSTGRES_PASSWORD=postgres
+EVAL_RUSTFS_ACCESS_KEY=admin
+EVAL_RUSTFS_SECRET_KEY=lobehub-eval-rustfs
+EVAL_RUSTFS_BUCKET=lobe
+EVAL_GATEWAY_SERVICE_TOKEN=lobehub-eval-gateway-token
+EVAL_LOBE_API_BASE_URL=http://host.docker.internal:3210
+
+# Matching LobeHub server environment. Keep APP_URL on localhost: QStash uses
+# host networking so it can deliver callbacks to this address from its container.
+APP_URL=http://localhost:3210
+INTERNAL_APP_URL=http://localhost:3210
+AUTH_SECRET=lobehub-eval-only-auth-secret-0001
+KEY_VAULTS_SECRET=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
+# The selected LobeHub agent owns its model and provider. If its provider
+# credential is not already stored in LobeHub, add the real provider variable.
+# Examples only:
+# OPENAI_API_KEY=
+# ANTHROPIC_API_KEY=
+# DEEPSEEK_API_KEY=
+# DEEPSEEK_PROXY_URL=https://api.deepseek.com/v1
+DATABASE_DRIVER=node
+DATABASE_URL=postgresql://postgres:postgres@localhost:15433/lobechat
+REDIS_URL=redis://localhost:6380
+S3_ENDPOINT=http://localhost:9100
+S3_PUBLIC_DOMAIN=http://localhost:9100
+S3_BUCKET=lobe
+S3_ENABLE_PATH_STYLE=1
+S3_ACCESS_KEY=admin
+S3_ACCESS_KEY_ID=admin
+S3_SECRET_ACCESS_KEY=lobehub-eval-rustfs
+S3_SET_ACL=0
+QSTASH_URL=http://127.0.0.1:8080
+QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0=
+QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
+QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
+DEVICE_GATEWAY_URL=http://localhost:8787
+DEVICE_GATEWAY_SERVICE_TOKEN=lobehub-eval-gateway-token
+AGENT_GATEWAY_URL=http://localhost:8788
+AGENT_GATEWAY_SERVICE_TOKEN=lobehub-eval-gateway-token
+AGENT_RUNTIME_BASE_URL=http://localhost:3210
+AGENT_RUNTIME_MODE=queue
+ENABLE_AGENT_FILE_TRACING=1
+ENABLE_AGENT_GATEWAY=1
+NEXT_PUBLIC_SERVICE_MODE=server
+TB_GRAPH_AGENT=1
+
+# Harbor task containers reach host-published services through the Docker bridge.
+# HARBOR_AGENT_GATEWAY_URL avoids colliding with LobeHub's host-local variable.
+LH_SERVER_URL=http://172.17.0.1:3210
+LH_GATEWAY_URL=http://172.17.0.1:8787
+HARBOR_AGENT_GATEWAY_URL=http://172.17.0.1:8788
+LH_AGENT_ID=

docker-compose/eval/bucket.config.json

@@ -0,0 +1,18 @@
+{
+  "Id": "",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": ["*"]
+      },
+      "Action": ["s3:GetObject"],
+      "NotAction": [],
+      "Resource": ["arn:aws:s3:::lobe/*"],
+      "NotResource": [],
+      "Condition": {}
+    }
+  ],
+  "Version": "2012-10-17"
+}

docker-compose/eval/docker-compose.yml

@@ -0,0 +1,141 @@
+name: ${EVAL_COMPOSE_PROJECT_NAME:-lobehub-eval}
+
+services:
+  postgresql:
+    image: paradedb/paradedb:latest-pg17
+    ports:
+      - '${EVAL_POSTGRES_PORT:-15433}:5432'
+    volumes:
+      - 'postgresql-data:/var/lib/postgresql/data'
+    command: ['postgres', '-c', 'shared_preload_libraries=pg_search']
+    environment:
+      POSTGRES_DB: ${EVAL_POSTGRES_DB:-lobechat}
+      POSTGRES_PASSWORD: ${EVAL_POSTGRES_PASSWORD:-postgres}
+    healthcheck:
+      test:
+        ['CMD-SHELL', 'pg_isready -U postgres -d ${EVAL_POSTGRES_DB:-lobechat}']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    restart: unless-stopped
+    networks:
+      - eval-network
+
+  redis:
+    image: redis:7-alpine
+    ports:
+      - '${EVAL_REDIS_PORT:-6380}:6379'
+    command: ['redis-server', '--save', '60', '1000', '--appendonly', 'yes']
+    volumes:
+      - 'redis-data:/data'
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    restart: unless-stopped
+    networks:
+      - eval-network
+
+  rustfs:
+    image: rustfs/rustfs:latest
+    ports:
+      - '${EVAL_RUSTFS_PORT:-9100}:9000'
+      - '${EVAL_RUSTFS_ADMIN_PORT:-9101}:9001'
+    environment:
+      RUSTFS_ACCESS_KEY: ${EVAL_RUSTFS_ACCESS_KEY:-admin}
+      RUSTFS_CONSOLE_ENABLE: 'true'
+      RUSTFS_SECRET_KEY: ${EVAL_RUSTFS_SECRET_KEY:-lobehub-eval-rustfs}
+    volumes:
+      - 'rustfs-data:/data'
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'wget -qO- http://localhost:9000/health >/dev/null 2>&1 || exit 1',
+        ]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    command:
+      [
+        '--access-key',
+        '${EVAL_RUSTFS_ACCESS_KEY:-admin}',
+        '--secret-key',
+        '${EVAL_RUSTFS_SECRET_KEY:-lobehub-eval-rustfs}',
+        '/data',
+      ]
+    restart: unless-stopped
+    networks:
+      - eval-network
+
+  rustfs-init:
+    image: rustfs/rc:latest
+    depends_on:
+      rustfs:
+        condition: service_healthy
+    volumes:
+      - './bucket.config.json:/bucket.config.json:ro'
+    entrypoint: /bin/sh
+    command: >-
+      -c 'set -eu;
+      rc alias set rustfs "http://rustfs:9000" "${EVAL_RUSTFS_ACCESS_KEY:-admin}" "${EVAL_RUSTFS_SECRET_KEY:-lobehub-eval-rustfs}";
+      rc mb "rustfs/${EVAL_RUSTFS_BUCKET:-lobe}" --ignore-existing;
+      rc anonymous set-json "/bucket.config.json" "rustfs/${EVAL_RUSTFS_BUCKET:-lobe}"'
+    restart: 'no'
+    networks:
+      - eval-network
+
+  # Host networking lets local QStash call an APP_URL on localhost without a tunnel.
+  # This eval stack targets the same Linux environment used by Harbor task containers.
+  qstash:
+    image: public.ecr.aws/upstash/qstash:2.37.18
+    network_mode: host
+    command:
+      [
+        'qstash',
+        'dev',
+        '-port=${EVAL_QSTASH_PORT:-8080}',
+        '-log-port=${EVAL_QSTASH_LOG_PORT:-8081}',
+      ]
+    restart: unless-stopped
+
+  # The unified image runs both gateways. Ports are intentionally swapped from
+  # its defaults to preserve the existing eval contract: device=8787, agent=8788.
+  gateway:
+    image: ghcr.io/lobehub/lobehub-gateway:0.3.2
+    depends_on:
+      rustfs-init:
+        condition: service_completed_successfully
+    ports:
+      - '${EVAL_DEVICE_GATEWAY_PORT:-8787}:8787'
+      - '${EVAL_AGENT_GATEWAY_PORT:-8788}:8788'
+    environment:
+      AGENT_PORT: 8788
+      DEVICE_PORT: 8787
+      JWKS_PUBLIC_KEY: ${EVAL_JWKS_PUBLIC_KEY:-}
+      LOBE_API_BASE_URL: ${EVAL_LOBE_API_BASE_URL:-http://host.docker.internal:3210}
+      SERVICE_TOKEN: ${EVAL_GATEWAY_SERVICE_TOKEN:-lobehub-eval-gateway-token}
+    extra_hosts:
+      - 'host.docker.internal:host-gateway'
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'wget -qO- http://127.0.0.1:8787/health >/dev/null && wget -qO- http://127.0.0.1:8788/health >/dev/null',
+        ]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    restart: unless-stopped
+    networks:
+      - eval-network
+
+volumes:
+  postgresql-data: {}
+  redis-data: {}
+  rustfs-data: {}
+
+networks:
+  eval-network:
+    driver: bridge