diff --git a/video_worker/README.md b/video_worker/README.md index dc6ed78..a9f0ca5 100644 --- a/video_worker/README.md +++ b/video_worker/README.md @@ -2,6 +2,13 @@ 一个本地单机视频生成 Worker,提供最小化 HTTP API:接收任务、按模式路由模型、单任务串行执行、输出统一结果目录。 +## 项目拆分(推荐部署方式) + +已拆分为两个可独立部署的目录: + +- `center_dispatch/`:中央调度项目(HTTP 管理 + WS 下发) +- `edge_node/`:边缘执行项目(本地推理 + 主动连中心) + ## 1. 项目说明 - 目标:边缘执行节点,不是完整平台。 diff --git a/video_worker/center_dispatch/.env.example b/video_worker/center_dispatch/.env.example new file mode 100644 index 0000000..44edda3 --- /dev/null +++ b/video_worker/center_dispatch/.env.example @@ -0,0 +1,24 @@ +EDGE_DISPATCH_HOST=0.0.0.0 +EDGE_DISPATCH_PORT=8060 +EDGE_MAX_DISPATCH_RECORDS=2000 + +PYTHON_IMAGE=docker.m.daocloud.io/library/python:3.10-slim +APT_MIRROR=mirrors.aliyun.com +PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ +PIP_TRUSTED_HOST=mirrors.aliyun.com + +AUTO_PULL_LATEST=true +GIT_REPO_URL= +GIT_BRANCH=master +GIT_CLONE_DEPTH=1 +GIT_PROJECT_SUBDIR=video_worker + +OSS_ENABLED=true +OSS_ENDPOINT=https://oss-cn-shanghai.aliyuncs.com +OSS_BUCKET=your-bucket +OSS_ACCESS_KEY_ID=your-ak +OSS_ACCESS_KEY_SECRET=your-sk +OSS_PUBLIC_BASE_URL=https://your-cdn-domain +OSS_PREFIX=video-worker + +LOG_LEVEL=INFO diff --git a/video_worker/center_dispatch/README.md b/video_worker/center_dispatch/README.md new file mode 100644 index 0000000..5a42b4a --- /dev/null +++ b/video_worker/center_dispatch/README.md @@ -0,0 +1,27 @@ +# Center Dispatch Project + +中央调度服务独立项目目录,仅负责: + +- 设备接入(`WS /ws/edge/{device_id}`) +- 上游 HTTP 下发任务(`POST /dispatch/generate`) +- 上游 HTTP 下发运维指令(`POST /devices/{device_id}/command`) +- 指令/任务状态查询(`GET /dispatch/{id}`、`GET /commands/{id}`) + +## Quick Start + +```bash +cd center_dispatch +bash scripts/start.sh +``` + +停止: + +```bash +bash scripts/stop.sh +``` + +## Notes + +- 首次运行会生成 `.env`,请至少配置: + - `GIT_REPO_URL`(开启 `AUTO_PULL_LATEST=true` 时必填) + - OSS 相关配置(`OSS_ENABLED=true` 时必填) diff --git a/video_worker/center_dispatch/docker-compose.yml b/video_worker/center_dispatch/docker-compose.yml new file mode 100644 index 0000000..37745cf --- /dev/null +++ b/video_worker/center_dispatch/docker-compose.yml @@ -0,0 +1,22 @@ +services: + edge-dispatch: + build: + context: .. + dockerfile: docker/edge-dispatch/Dockerfile + args: + PYTHON_IMAGE: ${PYTHON_IMAGE:-docker.m.daocloud.io/library/python:3.10-slim} + APT_MIRROR: ${APT_MIRROR:-mirrors.aliyun.com} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} + PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-mirrors.aliyun.com} + AUTO_PULL_LATEST: ${AUTO_PULL_LATEST:-true} + GIT_REPO_URL: ${GIT_REPO_URL:-} + GIT_BRANCH: ${GIT_BRANCH:-master} + GIT_CLONE_DEPTH: ${GIT_CLONE_DEPTH:-1} + GIT_PROJECT_SUBDIR: ${GIT_PROJECT_SUBDIR:-video_worker} + container_name: video-worker-center-dispatch + env_file: + - .env + command: ["python", "-m", "uvicorn", "app.edge_dispatch_service:app", "--host", "0.0.0.0", "--port", "${EDGE_DISPATCH_PORT:-8060}"] + ports: + - "${EDGE_DISPATCH_PORT:-8060}:${EDGE_DISPATCH_PORT:-8060}" + restart: unless-stopped diff --git a/video_worker/center_dispatch/scripts/start.sh b/video_worker/center_dispatch/scripts/start.sh new file mode 100644 index 0000000..f9f4b54 --- /dev/null +++ b/video_worker/center_dispatch/scripts/start.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROJECT_DIR="${ROOT_DIR}/center_dispatch" +cd "$PROJECT_DIR" + +if [ ! -f .env ]; then + cp .env.example .env + echo "[INFO] .env created from .env.example, please set GIT_REPO_URL/OSS config." +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "[ERROR] docker not found" + exit 1 +fi +if ! docker compose version >/dev/null 2>&1; then + echo "[ERROR] docker compose not available" + exit 1 +fi + +PORT=$(grep '^EDGE_DISPATCH_PORT=' .env | tail -n1 | cut -d'=' -f2- || true) +PORT="${PORT:-8060}" + +docker compose up -d --build +echo "[OK] center dispatch started" +echo "[INFO] health: curl http://127.0.0.1:${PORT}/health" diff --git a/video_worker/center_dispatch/scripts/stop.sh b/video_worker/center_dispatch/scripts/stop.sh new file mode 100644 index 0000000..cf1ac66 --- /dev/null +++ b/video_worker/center_dispatch/scripts/stop.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROJECT_DIR="${ROOT_DIR}/center_dispatch" +cd "$PROJECT_DIR" + +docker compose down +echo "[OK] center dispatch stopped" diff --git a/video_worker/edge_node/.env.example b/video_worker/edge_node/.env.example new file mode 100644 index 0000000..8014c05 --- /dev/null +++ b/video_worker/edge_node/.env.example @@ -0,0 +1,25 @@ +APP_HOST=127.0.0.1 +APP_PORT=8000 + +DISPATCH_WS_URL=ws://127.0.0.1:8060/ws/edge/edge-a4000-01 +WORKER_BASE_URL=http://127.0.0.1:8000 +EDGE_POLL_INTERVAL_SEC=1.0 + +EDGE_ALLOW_REMOTE_UPDATE=true +EDGE_ALLOW_REMOTE_RESTART=true + +OUTPUT_DIR=./outputs +RUNTIME_DIR=./runtime +SQLITE_PATH=./runtime/tasks.db + +LTX_MODEL_DIR=./models/ltx +HUNYUAN_MODEL_DIR=./models/hunyuan + +DEFAULT_WIDTH=832 +DEFAULT_HEIGHT=480 +DEFAULT_FPS=16 +DEFAULT_DURATION=5 +DEFAULT_STEPS_PREVIEW=8 +DEFAULT_STEPS_REFINE=12 + +LOG_LEVEL=INFO diff --git a/video_worker/edge_node/README.md b/video_worker/edge_node/README.md new file mode 100644 index 0000000..b2fad9e --- /dev/null +++ b/video_worker/edge_node/README.md @@ -0,0 +1,44 @@ +# Edge Node Project + +边缘设备独立项目目录(主动连接中心 WS,接收任务和运维指令)。 + +安全边界: + +- 边缘仅主动连接中心(`DISPATCH_WS_URL`) +- 边缘不提供外网入口,Worker 固定绑定 `127.0.0.1` + +## Linux / WSL + +```bash +cd edge_node +bash scripts/start.sh +``` + +停止与重启: + +```bash +bash scripts/stop.sh +bash scripts/restart.sh +``` + +## Windows PowerShell (WSL) + +```powershell +cd edge_node +.\scripts\wsl.ps1 -Action start +``` + +常用: + +```powershell +.\scripts\wsl.ps1 -Action status +.\scripts\wsl.ps1 -Action restart +.\scripts\wsl.ps1 -Action stop +``` + +## Config + +首次启动会生成 `edge_node/.env`。请至少设置: + +- `DISPATCH_WS_URL=ws://:8060/ws/edge/` +- `WORKER_BASE_URL=http://127.0.0.1:8000` diff --git a/video_worker/edge_node/scripts/restart.sh b/video_worker/edge_node/scripts/restart.sh new file mode 100644 index 0000000..8001329 --- /dev/null +++ b/video_worker/edge_node/scripts/restart.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)" +PROJECT_DIR="${ROOT_DIR}/edge_node" +cd "$ROOT_DIR" + +ENV_FILE="${PROJECT_DIR}/.env" +ENV_TEMPLATE_FILE="${PROJECT_DIR}/.env.example" +EDGE_RUNTIME_DIR="${PROJECT_DIR}/runtime" + +ENV_FILE="$ENV_FILE" ENV_TEMPLATE_FILE="$ENV_TEMPLATE_FILE" EDGE_RUNTIME_DIR="$EDGE_RUNTIME_DIR" bash scripts/restart_edge_device_local.sh diff --git a/video_worker/edge_node/scripts/start.sh b/video_worker/edge_node/scripts/start.sh new file mode 100644 index 0000000..ae1f995 --- /dev/null +++ b/video_worker/edge_node/scripts/start.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)" +PROJECT_DIR="${ROOT_DIR}/edge_node" +cd "$ROOT_DIR" + +ENV_FILE="${PROJECT_DIR}/.env" +ENV_TEMPLATE_FILE="${PROJECT_DIR}/.env.example" +EDGE_RUNTIME_DIR="${PROJECT_DIR}/runtime" + +ENV_FILE="$ENV_FILE" ENV_TEMPLATE_FILE="$ENV_TEMPLATE_FILE" EDGE_RUNTIME_DIR="$EDGE_RUNTIME_DIR" bash scripts/start_edge_device_local.sh diff --git a/video_worker/edge_node/scripts/stop.sh b/video_worker/edge_node/scripts/stop.sh new file mode 100644 index 0000000..4f09920 --- /dev/null +++ b/video_worker/edge_node/scripts/stop.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)" +PROJECT_DIR="${ROOT_DIR}/edge_node" +cd "$ROOT_DIR" + +EDGE_RUNTIME_DIR="${PROJECT_DIR}/runtime" +EDGE_RUNTIME_DIR="$EDGE_RUNTIME_DIR" bash scripts/stop_edge_device_local.sh diff --git a/video_worker/edge_node/scripts/wsl.ps1 b/video_worker/edge_node/scripts/wsl.ps1 new file mode 100644 index 0000000..f1397c7 --- /dev/null +++ b/video_worker/edge_node/scripts/wsl.ps1 @@ -0,0 +1,41 @@ +param( + [ValidateSet("start", "stop", "restart", "status")] + [string]$Action = "start", + [string]$Distro = "" +) + +$ErrorActionPreference = "Stop" + +$Root = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path)) +$ProjectRoot = Join-Path $Root "edge_node" + +if (!(Get-Command wsl -ErrorAction SilentlyContinue)) { + throw "WSL command not found. Please install WSL first." +} + +if ([string]::IsNullOrWhiteSpace($Distro)) { + $linuxRoot = (wsl -- wslpath -a "$Root").Trim() +} else { + $linuxRoot = (wsl -d $Distro -- wslpath -a "$Root").Trim() +} +if ([string]::IsNullOrWhiteSpace($linuxRoot)) { + throw "Failed to resolve WSL path for project root: $Root" +} + +switch ($Action) { + "start" { $target = "edge_node/scripts/start.sh" } + "stop" { $target = "edge_node/scripts/stop.sh" } + "restart" { $target = "edge_node/scripts/restart.sh" } + "status" { + $bash = "cd '$linuxRoot' && if [ -f edge_node/runtime/pids/worker.pid ] && kill -0 `$(cat edge_node/runtime/pids/worker.pid) >/dev/null 2>&1; then echo '[OK] worker running'; else echo '[INFO] worker not running'; fi; if [ -f edge_node/runtime/pids/edge_client.pid ] && kill -0 `$(cat edge_node/runtime/pids/edge_client.pid) >/dev/null 2>&1; then echo '[OK] edge_client running'; else echo '[INFO] edge_client not running'; fi" + if ([string]::IsNullOrWhiteSpace($Distro)) { wsl -- bash -lc "$bash" } else { wsl -d $Distro -- bash -lc "$bash" } + exit 0 + } +} + +$bashCommand = "cd '$linuxRoot' && chmod +x edge_node/scripts/start.sh edge_node/scripts/stop.sh edge_node/scripts/restart.sh scripts/start_edge_device_local.sh scripts/stop_edge_device_local.sh scripts/restart_edge_device_local.sh && bash $target" +if ([string]::IsNullOrWhiteSpace($Distro)) { + wsl -- bash -lc "$bashCommand" +} else { + wsl -d $Distro -- bash -lc "$bashCommand" +} diff --git a/video_worker/scripts/restart_edge_device_local.sh b/video_worker/scripts/restart_edge_device_local.sh index 981485a..05a7e84 100644 --- a/video_worker/scripts/restart_edge_device_local.sh +++ b/video_worker/scripts/restart_edge_device_local.sh @@ -4,6 +4,10 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -bash scripts/stop_edge_device_local.sh +EDGE_RUNTIME_DIR="${EDGE_RUNTIME_DIR:-runtime}" +ENV_FILE="${ENV_FILE:-.env}" +ENV_TEMPLATE_FILE="${ENV_TEMPLATE_FILE:-.env.example}" + +EDGE_RUNTIME_DIR="$EDGE_RUNTIME_DIR" bash scripts/stop_edge_device_local.sh sleep 1 -bash scripts/start_edge_device_local.sh +ENV_FILE="$ENV_FILE" ENV_TEMPLATE_FILE="$ENV_TEMPLATE_FILE" EDGE_RUNTIME_DIR="$EDGE_RUNTIME_DIR" bash scripts/start_edge_device_local.sh diff --git a/video_worker/scripts/start_edge_device_local.sh b/video_worker/scripts/start_edge_device_local.sh index a34c93c..11b53a4 100644 --- a/video_worker/scripts/start_edge_device_local.sh +++ b/video_worker/scripts/start_edge_device_local.sh @@ -4,8 +4,12 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -PID_DIR="runtime/pids" -LOG_DIR="runtime/logs" +ENV_FILE="${ENV_FILE:-.env}" +ENV_TEMPLATE_FILE="${ENV_TEMPLATE_FILE:-.env.example}" +EDGE_RUNTIME_DIR="${EDGE_RUNTIME_DIR:-runtime}" + +PID_DIR="${EDGE_RUNTIME_DIR}/pids" +LOG_DIR="${EDGE_RUNTIME_DIR}/logs" WORKER_PID_FILE="${PID_DIR}/worker.pid" EDGE_CLIENT_PID_FILE="${PID_DIR}/edge_client.pid" WORKER_LOG="${LOG_DIR}/worker.log" @@ -13,9 +17,9 @@ EDGE_CLIENT_LOG="${LOG_DIR}/edge_client.log" mkdir -p "$PID_DIR" "$LOG_DIR" -if [ ! -f .env ]; then - cp .env.example .env - echo "[INFO] .env missing, created from .env.example" +if [ ! -f "$ENV_FILE" ]; then + cp "$ENV_TEMPLATE_FILE" "$ENV_FILE" + echo "[INFO] ${ENV_FILE} missing, created from ${ENV_TEMPLATE_FILE}" fi if [ ! -d .venv ]; then @@ -25,11 +29,12 @@ fi source .venv/bin/activate set -a -source .env +source "$ENV_FILE" set +a -# Edge device should not expose a public ingress service. -APP_HOST="${APP_HOST:-127.0.0.1}" +# Edge device must not expose ingress service to external network. +# Force localhost bind even if APP_HOST is configured differently. +APP_HOST="127.0.0.1" APP_PORT="${APP_PORT:-8000}" DISPATCH_WS_URL="${DISPATCH_WS_URL:-}" WORKER_BASE_URL="${WORKER_BASE_URL:-http://127.0.0.1:${APP_PORT}}" diff --git a/video_worker/scripts/stop_edge_device_local.sh b/video_worker/scripts/stop_edge_device_local.sh index 6c49253..c73267e 100644 --- a/video_worker/scripts/stop_edge_device_local.sh +++ b/video_worker/scripts/stop_edge_device_local.sh @@ -4,7 +4,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -PID_DIR="runtime/pids" +EDGE_RUNTIME_DIR="${EDGE_RUNTIME_DIR:-runtime}" +PID_DIR="${EDGE_RUNTIME_DIR}/pids" WORKER_PID_FILE="${PID_DIR}/worker.pid" EDGE_CLIENT_PID_FILE="${PID_DIR}/edge_client.pid"