feat: 初始化零工后端代码

This commit is contained in:
Daniel
2026-04-01 14:19:25 +08:00
parent c6fabe262c
commit 84f8be7c0e
41 changed files with 2813 additions and 147 deletions

View File

@@ -1,23 +1,33 @@
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Path
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.db.session import get_db
from app.domain.schemas import (
AIObservabilityResponse,
BootstrapResponse,
ExplainResponse,
ExtractResponse,
ExtractTextRequest,
HealthStatus,
IngestJobRequest,
IngestAsyncResponse,
IngestWorkerRequest,
JobCard,
ListResponse,
MatchFeedbackRequest,
MatchAsyncJobsRequest,
MatchAsyncResponse,
MatchAsyncWorkersRequest,
MatchJobsRequest,
MatchResponse,
MatchWeightResponse,
MatchWorkersRequest,
QueueStatusResponse,
SystemOpsResponse,
WorkerCard,
)
from app.repositories.job_repository import JobRepository
@@ -25,14 +35,23 @@ from app.repositories.worker_repository import WorkerRepository
from app.services.card_mapper import job_to_card, worker_to_card
from app.services.extraction_service import ExtractionService
from app.services.ingest_service import IngestService
from app.services.llm_client import LLMClient
from app.services.matching_service import MatchingService
from app.services.cache_service import get_match_cache, get_query_cache
from app.services.runtime_state import get_ingest_queue, get_match_queue, get_traffic_guard
from app.services.rag.lightrag_adapter import LightRAGAdapter
router = APIRouter()
@router.get("/health", response_model=HealthStatus)
@router.get(
"/health",
response_model=HealthStatus,
tags=["系统"],
summary="服务健康检查",
description="检查 API 服务、数据库与 RAG 检索组件状态。",
)
def health(db: Session = Depends(get_db)) -> HealthStatus:
settings = get_settings()
db_status = "ok"
@@ -48,32 +67,119 @@ def health(db: Session = Depends(get_db)) -> HealthStatus:
return HealthStatus(service="ok", database=db_status, rag=rag_status, timestamp=datetime.now().astimezone())
@router.post("/poc/extract/job", response_model=ExtractResponse)
@router.post(
"/poc/extract/job",
response_model=ExtractResponse,
tags=["抽取"],
summary="岗位文本抽取",
description="将岗位自然语言文本抽取为结构化 JobCard。",
)
def extract_job(payload: ExtractTextRequest) -> ExtractResponse:
return ExtractionService().extract_job(payload.text)
@router.post("/poc/extract/worker", response_model=ExtractResponse)
@router.post(
"/poc/extract/worker",
response_model=ExtractResponse,
tags=["抽取"],
summary="工人文本抽取",
description="将工人自然语言文本抽取为结构化 WorkerCard。",
)
def extract_worker(payload: ExtractTextRequest) -> ExtractResponse:
return ExtractionService().extract_worker(payload.text)
@router.post("/poc/ingest/job", response_model=JobCard)
@router.post(
"/poc/ingest/job",
response_model=JobCard,
tags=["入库"],
summary="岗位入库",
description="写入或更新岗位卡片,并同步更新检索索引。",
)
def ingest_job(payload: IngestJobRequest, db: Session = Depends(get_db)) -> JobCard:
return IngestService(db).ingest_job(payload.job)
@router.post("/poc/ingest/worker", response_model=WorkerCard)
@router.post(
"/poc/ingest/worker",
response_model=WorkerCard,
tags=["入库"],
summary="工人入库",
description="写入或更新工人卡片,并同步更新检索索引。",
)
def ingest_worker(payload: IngestWorkerRequest, db: Session = Depends(get_db)) -> WorkerCard:
return IngestService(db).ingest_worker(payload.worker)
@router.post("/poc/ingest/bootstrap")
def bootstrap(db: Session = Depends(get_db)):
@router.post(
"/poc/ingest/job/async",
response_model=IngestAsyncResponse,
tags=["入库"],
summary="岗位异步入库",
description="将岗位入库请求写入异步队列,快速返回任务 ID。",
)
def ingest_job_async(payload: IngestJobRequest) -> IngestAsyncResponse:
settings = get_settings()
if not settings.ingest_async_enabled:
raise HTTPException(status_code=400, detail="异步入库未开启")
queue = get_ingest_queue()
try:
task_id = queue.enqueue_job(payload.job)
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
return IngestAsyncResponse(task_id=task_id, status=queue.task_status(task_id))
@router.post(
"/poc/ingest/worker/async",
response_model=IngestAsyncResponse,
tags=["入库"],
summary="工人异步入库",
description="将工人入库请求写入异步队列,快速返回任务 ID。",
)
def ingest_worker_async(payload: IngestWorkerRequest) -> IngestAsyncResponse:
settings = get_settings()
if not settings.ingest_async_enabled:
raise HTTPException(status_code=400, detail="异步入库未开启")
queue = get_ingest_queue()
try:
task_id = queue.enqueue_worker(payload.worker)
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
return IngestAsyncResponse(task_id=task_id, status=queue.task_status(task_id))
@router.get(
"/poc/ingest/queue/{task_id}",
response_model=IngestAsyncResponse,
tags=["入库"],
summary="异步入库任务状态",
description="根据 task_id 查询异步入库任务状态。",
)
def ingest_task_status(task_id: str) -> IngestAsyncResponse:
queue = get_ingest_queue()
return IngestAsyncResponse(task_id=task_id, status=queue.task_status(task_id))
@router.post(
"/poc/ingest/bootstrap",
response_model=BootstrapResponse,
tags=["入库"],
summary="样本数据初始化",
description="导入内置样本数据(岗位、工人、技能、类目、区域)并构建检索数据。",
)
def bootstrap(db: Session = Depends(get_db)) -> BootstrapResponse:
return IngestService(db).bootstrap()
@router.post("/poc/match/workers", response_model=MatchResponse)
@router.post(
"/poc/match/workers",
response_model=MatchResponse,
tags=["匹配"],
summary="岗位匹配工人",
description="支持通过 job_id 或内联 job 进行匹配,返回 top_n 条结果。",
responses={404: {"description": "岗位不存在"}},
)
def match_workers(payload: MatchWorkersRequest, db: Session = Depends(get_db)) -> MatchResponse:
service = MatchingService(db)
source = payload.job
@@ -85,7 +191,14 @@ def match_workers(payload: MatchWorkersRequest, db: Session = Depends(get_db)) -
return MatchResponse(items=service.match_workers(source, payload.top_n))
@router.post("/poc/match/jobs", response_model=MatchResponse)
@router.post(
"/poc/match/jobs",
response_model=MatchResponse,
tags=["匹配"],
summary="工人匹配岗位",
description="支持通过 worker_id 或内联 worker 进行匹配,返回 top_n 条结果。",
responses={404: {"description": "工人不存在"}},
)
def match_jobs(payload: MatchJobsRequest, db: Session = Depends(get_db)) -> MatchResponse:
service = MatchingService(db)
source = payload.worker
@@ -97,37 +210,245 @@ def match_jobs(payload: MatchJobsRequest, db: Session = Depends(get_db)) -> Matc
return MatchResponse(items=service.match_jobs(source, payload.top_n))
@router.get("/poc/match/explain/{match_id}", response_model=ExplainResponse)
def explain_match(match_id: str, db: Session = Depends(get_db)) -> ExplainResponse:
@router.post(
"/poc/match/workers/async",
response_model=MatchAsyncResponse,
tags=["匹配"],
summary="岗位异步匹配工人",
description="将匹配任务放入队列异步计算,适合高并发削峰。",
)
def match_workers_async(payload: MatchAsyncWorkersRequest) -> MatchAsyncResponse:
settings = get_settings()
if not settings.match_async_enabled:
raise HTTPException(status_code=400, detail="异步匹配未开启")
queue = get_match_queue()
try:
task_id = queue.enqueue_workers(payload.job_id, payload.top_n)
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
return MatchAsyncResponse(task_id=task_id, status=queue.task_status(task_id))
@router.post(
"/poc/match/jobs/async",
response_model=MatchAsyncResponse,
tags=["匹配"],
summary="工人异步匹配岗位",
description="将匹配任务放入队列异步计算,适合高并发削峰。",
)
def match_jobs_async(payload: MatchAsyncJobsRequest) -> MatchAsyncResponse:
settings = get_settings()
if not settings.match_async_enabled:
raise HTTPException(status_code=400, detail="异步匹配未开启")
queue = get_match_queue()
try:
task_id = queue.enqueue_jobs(payload.worker_id, payload.top_n)
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
return MatchAsyncResponse(task_id=task_id, status=queue.task_status(task_id))
@router.get(
"/poc/match/queue/{task_id}",
response_model=MatchAsyncResponse,
tags=["匹配"],
summary="异步匹配任务状态",
description="根据 task_id 查询异步匹配任务状态,完成后返回匹配结果。",
)
def match_task_status(task_id: str) -> MatchAsyncResponse:
queue = get_match_queue()
status = queue.task_status(task_id)
items = queue.task_result(task_id)
return MatchAsyncResponse(task_id=task_id, status=status, items=items)
@router.get(
"/poc/match/explain/{match_id}",
response_model=ExplainResponse,
tags=["匹配"],
summary="匹配结果解释",
description="根据 match_id 获取匹配明细和解释理由。",
responses={404: {"description": "匹配记录不存在"}},
)
def explain_match(
match_id: str = Path(..., description="匹配记录 ID"),
db: Session = Depends(get_db),
) -> ExplainResponse:
match = MatchingService(db).explain(match_id)
if match is None:
raise HTTPException(status_code=404, detail="匹配记录不存在")
return ExplainResponse(match=match)
@router.get("/poc/jobs", response_model=ListResponse)
@router.post(
"/poc/match/feedback",
response_model=MatchWeightResponse,
tags=["匹配"],
summary="匹配反馈学习",
description="提交单条匹配的接受/拒绝反馈,用于在线更新排序权重。",
responses={404: {"description": "匹配记录不存在"}},
)
def feedback_match(payload: MatchFeedbackRequest, db: Session = Depends(get_db)) -> MatchWeightResponse:
service = MatchingService(db)
weights = service.feedback(payload.match_id, payload.accepted)
if weights is None:
raise HTTPException(status_code=404, detail="匹配记录不存在")
return MatchWeightResponse(weights=weights, learning_enabled=get_settings().ranking_learning_enabled)
@router.get(
"/poc/match/weights",
response_model=MatchWeightResponse,
tags=["匹配"],
summary="当前排序权重",
description="查看当前生效的排序权重(默认权重或学习后的权重)。",
)
def get_match_weights(db: Session = Depends(get_db)) -> MatchWeightResponse:
service = MatchingService(db)
return MatchWeightResponse(weights=service.current_weights(), learning_enabled=get_settings().ranking_learning_enabled)
@router.get(
"/poc/ops/ai/metrics",
response_model=AIObservabilityResponse,
tags=["系统"],
summary="AI 观测指标",
description="返回 AI 调用的限流、熔断、降级与 fallback 命中率指标。",
)
def ai_metrics() -> AIObservabilityResponse:
metrics = LLMClient(get_settings()).metrics()
return AIObservabilityResponse(metrics=metrics)
@router.get(
"/poc/ops/system/metrics",
response_model=SystemOpsResponse,
tags=["系统"],
summary="系统运行指标",
description="返回全局流量护栏、缓存与异步队列指标。",
)
def system_metrics() -> SystemOpsResponse:
queue_stats = get_ingest_queue().stats()
match_queue_stats = get_match_queue().stats()
match_cache_stats = get_match_cache().stats()
query_cache_stats = get_query_cache().stats()
return SystemOpsResponse(
traffic=get_traffic_guard().snapshot(),
cache={
"backend": match_cache_stats.get("backend", "memory"),
"match_hit_rate": match_cache_stats.get("hit_rate", 0.0),
"query_hit_rate": query_cache_stats.get("hit_rate", 0.0),
"match_size": int(match_cache_stats.get("size", 0)),
"query_size": int(query_cache_stats.get("size", 0)),
},
ingest_queue=QueueStatusResponse(
queued=queue_stats["queued"],
processed=queue_stats["processed"],
failed=queue_stats["failed"],
),
match_queue=QueueStatusResponse(
queued=match_queue_stats["queued"],
processed=match_queue_stats["processed"],
failed=match_queue_stats["failed"],
),
)
@router.get(
"/poc/jobs",
response_model=ListResponse,
tags=["查询"],
summary="岗位列表查询",
description="查询岗位列表,当前返回全量数据。",
)
def list_jobs(db: Session = Depends(get_db)) -> ListResponse:
settings = get_settings()
cache = get_query_cache()
cache_key = "jobs:list"
if settings.query_cache_enabled:
cached = cache.get(cache_key)
if cached is not None:
return ListResponse(items=cached["items"], total=cached["total"])
items = [job_to_card(item).model_dump(mode="json") for item in JobRepository(db).list()]
return ListResponse(items=items, total=len(items))
result = ListResponse(items=items, total=len(items))
if settings.query_cache_enabled:
cache.set(cache_key, result.model_dump(mode="json"))
return result
@router.get("/poc/workers", response_model=ListResponse)
@router.get(
"/poc/workers",
response_model=ListResponse,
tags=["查询"],
summary="工人列表查询",
description="查询工人列表,当前返回全量数据。",
)
def list_workers(db: Session = Depends(get_db)) -> ListResponse:
settings = get_settings()
cache = get_query_cache()
cache_key = "workers:list"
if settings.query_cache_enabled:
cached = cache.get(cache_key)
if cached is not None:
return ListResponse(items=cached["items"], total=cached["total"])
items = [worker_to_card(item).model_dump(mode="json") for item in WorkerRepository(db).list()]
return ListResponse(items=items, total=len(items))
result = ListResponse(items=items, total=len(items))
if settings.query_cache_enabled:
cache.set(cache_key, result.model_dump(mode="json"))
return result
@router.get("/poc/jobs/{job_id}", response_model=JobCard)
def get_job(job_id: str, db: Session = Depends(get_db)) -> JobCard:
@router.get(
"/poc/jobs/{job_id}",
response_model=JobCard,
tags=["查询"],
summary="岗位详情查询",
description="根据岗位 ID 查询单个岗位详情。",
responses={404: {"description": "岗位不存在"}},
)
def get_job(
job_id: str = Path(..., description="岗位 ID"),
db: Session = Depends(get_db),
) -> JobCard:
settings = get_settings()
cache = get_query_cache()
cache_key = f"jobs:detail:{job_id}"
if settings.query_cache_enabled:
cached = cache.get(cache_key)
if cached is not None:
return JobCard(**cached)
item = JobRepository(db).get(job_id)
if item is None:
raise HTTPException(status_code=404, detail="岗位不存在")
return job_to_card(item)
result = job_to_card(item)
if settings.query_cache_enabled:
cache.set(cache_key, result.model_dump(mode="json"))
return result
@router.get("/poc/workers/{worker_id}", response_model=WorkerCard)
def get_worker(worker_id: str, db: Session = Depends(get_db)) -> WorkerCard:
@router.get(
"/poc/workers/{worker_id}",
response_model=WorkerCard,
tags=["查询"],
summary="工人详情查询",
description="根据工人 ID 查询单个工人详情。",
responses={404: {"description": "工人不存在"}},
)
def get_worker(
worker_id: str = Path(..., description="工人 ID"),
db: Session = Depends(get_db),
) -> WorkerCard:
settings = get_settings()
cache = get_query_cache()
cache_key = f"workers:detail:{worker_id}"
if settings.query_cache_enabled:
cached = cache.get(cache_key)
if cached is not None:
return WorkerCard(**cached)
item = WorkerRepository(db).get(worker_id)
if item is None:
raise HTTPException(status_code=404, detail="工人不存在")
return worker_to_card(item)
result = worker_to_card(item)
if settings.query_cache_enabled:
cache.set(cache_key, result.model_dump(mode="json"))
return result

View File

@@ -17,18 +17,39 @@ class Settings(BaseSettings):
app_host: str = "0.0.0.0"
app_port: int = 8000
log_level: str = "INFO"
app_rate_limit_per_minute: int = 1200
app_circuit_breaker_error_rate: float = 0.5
app_circuit_breaker_min_requests: int = 50
app_circuit_breaker_window_seconds: int = 60
app_circuit_breaker_cooldown_seconds: int = 30
alert_webhook_url: str | None = None
database_url: str = "postgresql+psycopg://gig:gig@postgres:5432/gig_poc"
database_pool_size: int = 20
database_max_overflow: int = 30
database_pool_timeout: int = 30
qdrant_url: str = "http://qdrant:6333"
qdrant_collection: str = "gig_poc_entities"
vector_size: int = 64
llm_enabled: bool = False
llm_base_url: str | None = None
llm_fallback_base_urls: list[str] = Field(default_factory=list)
llm_api_key: str | None = None
llm_model: str = "gpt-5.4"
extraction_llm_max_retries: int = 2
embedding_backend: str = "hash" # hash | openai_compatible
embedding_enabled: bool = False
embedding_base_url: str | None = None
embedding_fallback_base_urls: list[str] = Field(default_factory=list)
embedding_api_key: str | None = None
embedding_model: str = "text-embedding-3-small"
embedding_vector_size: int = 1536
ai_request_timeout_seconds: float = 30.0
ai_rate_limit_per_minute: int = 120
ai_circuit_breaker_fail_threshold: int = 5
ai_circuit_breaker_cooldown_seconds: int = 30
bootstrap_jobs: int = 100
bootstrap_workers: int = 300
@@ -38,12 +59,27 @@ class Settings(BaseSettings):
prompt_dir: Path = Field(default=ROOT_DIR / "packages" / "prompts")
sample_data_dir: Path = Field(default=ROOT_DIR / "packages" / "sample-data")
shared_types_dir: Path = Field(default=ROOT_DIR / "packages" / "shared-types")
data_dir: Path = Field(default=ROOT_DIR / "data")
match_weights_path: Path = Field(default=ROOT_DIR / "data" / "match_weights.json")
score_skill_weight: float = 0.35
score_region_weight: float = 0.20
score_time_weight: float = 0.15
score_experience_weight: float = 0.15
score_reliability_weight: float = 0.15
ranking_learning_enabled: bool = True
ranking_learning_rate: float = 0.08
cache_backend: str = "memory" # memory | redis
redis_url: str = "redis://redis:6379/0"
redis_prefix: str = "gig_poc"
match_cache_enabled: bool = True
match_cache_ttl_seconds: int = 30
query_cache_enabled: bool = True
query_cache_ttl_seconds: int = 20
ingest_async_enabled: bool = True
ingest_queue_max_size: int = 10000
match_async_enabled: bool = True
match_queue_max_size: int = 10000
@lru_cache

View File

@@ -7,7 +7,14 @@ from app.core.config import get_settings
settings = get_settings()
engine = create_engine(settings.database_url, future=True, pool_pre_ping=True)
engine = create_engine(
settings.database_url,
future=True,
pool_pre_ping=True,
pool_size=settings.database_pool_size,
max_overflow=settings.database_max_overflow,
pool_timeout=settings.database_pool_timeout,
)
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)

View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field, field_validator, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class SalaryType(str, Enum):
@@ -19,82 +19,89 @@ class SourceType(str, Enum):
class Salary(BaseModel):
type: SalaryType = SalaryType.daily
amount: float = 0
currency: str = "CNY"
type: SalaryType = Field(default=SalaryType.daily, description="薪资类型daily/hourly/monthly/task")
amount: float = Field(default=0, description="薪资金额")
currency: str = Field(default="CNY", description="货币类型,默认 CNY")
class SkillScore(BaseModel):
name: str
score: float = Field(ge=0, le=1)
name: str = Field(description="技能名称")
score: float = Field(ge=0, le=1, description="技能熟练度,范围 0~1")
class JobCard(BaseModel):
job_id: str
title: str
category: str
description: str
skills: list[str] = Field(default_factory=list)
city: str
region: str
location_detail: str
start_time: datetime
duration_hours: float = Field(gt=0)
headcount: int = Field(gt=0)
salary: Salary
work_mode: str
tags: list[str] = Field(default_factory=list)
confidence: float = Field(ge=0, le=1)
job_id: str = Field(description="岗位唯一 ID")
title: str = Field(description="岗位标题")
category: str = Field(description="岗位类别")
description: str = Field(description="岗位描述")
skills: list[str] = Field(default_factory=list, description="岗位技能要求列表")
city: str = Field(description="城市")
region: str = Field(description="区域")
location_detail: str = Field(description="详细地点描述")
start_time: datetime = Field(description="岗位开始时间ISO-8601")
duration_hours: float = Field(gt=0, description="工时(小时),必须大于 0")
headcount: int = Field(gt=0, description="招聘人数,必须大于 0")
salary: Salary = Field(description="薪资信息")
work_mode: str = Field(description="工作模式,如兼职、全职、活动")
tags: list[str] = Field(default_factory=list, description="业务标签列表")
confidence: float = Field(ge=0, le=1, description="数据置信度,范围 0~1")
class WorkerCard(BaseModel):
worker_id: str
name: str
description: str
skills: list[SkillScore] = Field(default_factory=list)
cities: list[str] = Field(default_factory=list)
regions: list[str] = Field(default_factory=list)
availability: list[str] = Field(default_factory=list)
experience_tags: list[str] = Field(default_factory=list)
reliability_score: float = Field(ge=0, le=1)
profile_completion: float = Field(ge=0, le=1)
confidence: float = Field(ge=0, le=1)
worker_id: str = Field(description="工人唯一 ID")
name: str = Field(description="工人姓名或昵称")
description: str = Field(description="工人自我描述")
skills: list[SkillScore] = Field(default_factory=list, description="技能及熟练度列表")
cities: list[str] = Field(default_factory=list, description="可接单城市列表")
regions: list[str] = Field(default_factory=list, description="可接单区域列表")
availability: list[str] = Field(default_factory=list, description="可上岗时间描述")
experience_tags: list[str] = Field(default_factory=list, description="经验标签列表")
reliability_score: float = Field(ge=0, le=1, description="履约可靠性分,范围 0~1")
profile_completion: float = Field(ge=0, le=1, description="档案完善度,范围 0~1")
confidence: float = Field(ge=0, le=1, description="数据置信度,范围 0~1")
class MatchBreakdown(BaseModel):
skill_score: float = Field(ge=0, le=1)
region_score: float = Field(ge=0, le=1)
time_score: float = Field(ge=0, le=1)
experience_score: float = Field(ge=0, le=1)
reliability_score: float = Field(ge=0, le=1)
skill_score: float = Field(ge=0, le=1, description="技能匹配分,范围 0~1")
region_score: float = Field(ge=0, le=1, description="地域匹配分,范围 0~1")
time_score: float = Field(ge=0, le=1, description="时间匹配分,范围 0~1")
experience_score: float = Field(ge=0, le=1, description="经验匹配分,范围 0~1")
reliability_score: float = Field(ge=0, le=1, description="可靠性匹配分,范围 0~1")
class MatchResult(BaseModel):
match_id: str
source_type: SourceType
source_id: str
target_id: str
match_score: float = Field(ge=0, le=1)
breakdown: MatchBreakdown
reasons: list[str] = Field(default_factory=list, min_length=3)
match_id: str = Field(description="匹配记录 ID")
source_type: SourceType = Field(description="匹配方向job_to_worker 或 worker_to_job")
source_id: str = Field(description="源实体 ID")
target_id: str = Field(description="目标实体 ID")
match_score: float = Field(ge=0, le=1, description="综合匹配分,范围 0~1")
breakdown: MatchBreakdown = Field(description="多维打分拆解")
reasons: list[str] = Field(default_factory=list, min_length=3, description="匹配理由,至少 3 条")
class ExtractTextRequest(BaseModel):
text: str = Field(min_length=5)
text: str = Field(min_length=5, description="待抽取的自然语言文本,最少 5 个字符")
model_config = ConfigDict(
json_schema_extra={
"example": {
"text": "明天下午南山会展中心需要2个签到协助5小时150/人,女生优先",
}
}
)
class IngestJobRequest(BaseModel):
job: JobCard
job: JobCard = Field(description="岗位卡片对象")
class IngestWorkerRequest(BaseModel):
worker: WorkerCard
worker: WorkerCard = Field(description="工人卡片对象")
class MatchWorkersRequest(BaseModel):
job_id: str | None = None
job: JobCard | None = None
top_n: int = Field(default=10, ge=1, le=50)
job_id: str | None = Field(default=None, description="岗位 ID与 job 二选一)")
job: JobCard | None = Field(default=None, description="内联岗位对象(与 job_id 二选一)")
top_n: int = Field(default=10, ge=1, le=50, description="返回条数,范围 1~50")
@model_validator(mode="after")
def validate_source(self) -> "MatchWorkersRequest":
@@ -104,9 +111,9 @@ class MatchWorkersRequest(BaseModel):
class MatchJobsRequest(BaseModel):
worker_id: str | None = None
worker: WorkerCard | None = None
top_n: int = Field(default=10, ge=1, le=50)
worker_id: str | None = Field(default=None, description="工人 ID与 worker 二选一)")
worker: WorkerCard | None = Field(default=None, description="内联工人对象(与 worker_id 二选一)")
top_n: int = Field(default=10, ge=1, le=50, description="返回条数,范围 1~50")
@model_validator(mode="after")
def validate_source(self) -> "MatchJobsRequest":
@@ -116,38 +123,86 @@ class MatchJobsRequest(BaseModel):
class ExtractResponse(BaseModel):
success: bool
data: JobCard | WorkerCard | None = None
errors: list[str] = Field(default_factory=list)
missing_fields: list[str] = Field(default_factory=list)
success: bool = Field(description="抽取是否成功")
data: JobCard | WorkerCard | None = Field(default=None, description="抽取结果对象,可能为空")
errors: list[str] = Field(default_factory=list, description="错误信息列表")
missing_fields: list[str] = Field(default_factory=list, description="缺失字段列表")
class BootstrapResponse(BaseModel):
jobs: int
workers: int
skills: int
categories: int
regions: int
jobs: int = Field(description="导入岗位数量")
workers: int = Field(description="导入工人数量")
skills: int = Field(description="技能词条数量")
categories: int = Field(description="类目数量")
regions: int = Field(description="区域数量")
class HealthStatus(BaseModel):
service: str
database: str
rag: str
timestamp: datetime
service: str = Field(description="服务状态,通常为 ok")
database: str = Field(description="数据库状态ok 或 error")
rag: str = Field(description="RAG 组件状态ok 或 error")
timestamp: datetime = Field(description="服务端当前时间")
class ListResponse(BaseModel):
items: list[dict]
total: int
items: list[dict] = Field(description="列表项")
total: int = Field(description="总数")
class MatchResponse(BaseModel):
items: list[MatchResult]
items: list[MatchResult] = Field(description="匹配结果列表")
class ExplainResponse(BaseModel):
match: MatchResult
match: MatchResult = Field(description="单条匹配结果详情")
class MatchFeedbackRequest(BaseModel):
match_id: str = Field(description="匹配记录 ID")
accepted: bool = Field(description="反馈是否接受该推荐")
class MatchWeightResponse(BaseModel):
weights: dict[str, float] = Field(description="当前生效的排序权重")
learning_enabled: bool = Field(description="是否开启在线学习")
class AIObservabilityResponse(BaseModel):
metrics: dict[str, float | int] = Field(description="AI 调用观测指标")
class IngestAsyncResponse(BaseModel):
task_id: str = Field(description="异步任务 ID")
status: str = Field(description="任务状态")
class QueueStatusResponse(BaseModel):
queued: int = Field(description="当前队列中任务数量")
processed: int = Field(description="历史处理成功数量")
failed: int = Field(description="历史处理失败数量")
class MatchAsyncWorkersRequest(BaseModel):
job_id: str = Field(description="岗位 ID")
top_n: int = Field(default=10, ge=1, le=50, description="返回条数,范围 1~50")
class MatchAsyncJobsRequest(BaseModel):
worker_id: str = Field(description="工人 ID")
top_n: int = Field(default=10, ge=1, le=50, description="返回条数,范围 1~50")
class MatchAsyncResponse(BaseModel):
task_id: str = Field(description="异步任务 ID")
status: str = Field(description="任务状态")
items: list[MatchResult] | None = Field(default=None, description="任务完成后返回的匹配结果")
class SystemOpsResponse(BaseModel):
traffic: dict[str, float | int] = Field(description="全局流量护栏与错误窗口指标")
cache: dict[str, float | int | str] = Field(description="缓存命中与大小")
ingest_queue: QueueStatusResponse = Field(description="异步入库队列状态")
match_queue: QueueStatusResponse = Field(description="异步匹配队列状态")
class PromptOutput(BaseModel):

View File

@@ -1,6 +1,8 @@
from contextlib import asynccontextmanager
from time import perf_counter
from fastapi import FastAPI
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from app.api.routes import router
@@ -9,6 +11,7 @@ from app.core.logging import configure_logging, logger
from app.db.base import Base
from app.db.session import engine
from app.services.rag.lightrag_adapter import LightRAGAdapter
from app.services.runtime_state import get_ingest_queue, get_match_queue, get_traffic_guard
settings = get_settings()
@@ -18,14 +21,33 @@ configure_logging(settings.log_level)
@asynccontextmanager
async def lifespan(_: FastAPI):
Base.metadata.create_all(bind=engine)
get_ingest_queue().start()
get_match_queue().start()
try:
LightRAGAdapter(settings).ensure_ready()
except Exception:
logger.exception("Qdrant initialization skipped during startup")
yield
get_ingest_queue().stop()
get_match_queue().stop()
app = FastAPI(title=settings.app_name, lifespan=lifespan)
app = FastAPI(
title=settings.app_name,
description=(
"Gig POC 接口文档。\n\n"
"接口分组:系统、抽取、入库、匹配、查询。\n"
"完整业务说明请参考项目文档 `docs/API.md`。"
),
openapi_tags=[
{"name": "系统", "description": "服务与依赖组件状态检查接口"},
{"name": "抽取", "description": "自然语言文本抽取为结构化卡片"},
{"name": "入库", "description": "结构化岗位/工人数据写入与初始化"},
{"name": "匹配", "description": "岗位与工人双向匹配及结果解释"},
{"name": "查询", "description": "岗位/工人列表与详情查询"},
],
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
@@ -33,4 +55,23 @@ app.add_middleware(
allow_methods=["*"],
allow_headers=["*"],
)
@app.middleware("http")
async def traffic_guard_middleware(request: Request, call_next):
guard = get_traffic_guard()
allowed, reason = guard.allow(request.url.path)
if not allowed:
status_code = 429 if reason == "rate_limited" else 503
return JSONResponse(status_code=status_code, content={"detail": reason})
start = perf_counter()
try:
response = await call_next(request)
except Exception:
guard.record(500, (perf_counter() - start) * 1000)
raise
guard.record(response.status_code, (perf_counter() - start) * 1000)
return response
app.include_router(router)

View File

@@ -0,0 +1,87 @@
from __future__ import annotations
import time
from dataclasses import dataclass
from threading import Lock
from app.core.config import Settings
@dataclass
class EndpointState:
current_minute: int = 0
minute_count: int = 0
consecutive_failures: int = 0
circuit_open_until: float = 0.0
class AIGuard:
_lock = Lock()
_endpoint_states: dict[str, EndpointState] = {}
_metrics = {
"requests_total": 0,
"success_total": 0,
"fail_total": 0,
"fallback_total": 0,
"rate_limited_total": 0,
"circuit_open_total": 0,
"endpoint_failover_total": 0,
}
def __init__(self, settings: Settings):
self.settings = settings
def allow_request(self, endpoint: str) -> tuple[bool, str]:
now = time.time()
now_minute = int(now // 60)
with self._lock:
state = self._endpoint_states.setdefault(endpoint, EndpointState())
if state.circuit_open_until > now:
self._metrics["circuit_open_total"] += 1
return False, "circuit_open"
if state.current_minute != now_minute:
state.current_minute = now_minute
state.minute_count = 0
if state.minute_count >= self.settings.ai_rate_limit_per_minute:
self._metrics["rate_limited_total"] += 1
return False, "rate_limited"
state.minute_count += 1
self._metrics["requests_total"] += 1
return True, "ok"
def record_success(self, endpoint: str) -> None:
with self._lock:
state = self._endpoint_states.setdefault(endpoint, EndpointState())
state.consecutive_failures = 0
state.circuit_open_until = 0.0
self._metrics["success_total"] += 1
def record_failure(self, endpoint: str) -> None:
with self._lock:
state = self._endpoint_states.setdefault(endpoint, EndpointState())
state.consecutive_failures += 1
self._metrics["fail_total"] += 1
if state.consecutive_failures >= self.settings.ai_circuit_breaker_fail_threshold:
state.circuit_open_until = time.time() + self.settings.ai_circuit_breaker_cooldown_seconds
state.consecutive_failures = 0
def record_fallback(self) -> None:
with self._lock:
self._metrics["fallback_total"] += 1
def record_failover(self) -> None:
with self._lock:
self._metrics["endpoint_failover_total"] += 1
def snapshot(self) -> dict:
with self._lock:
requests_total = self._metrics["requests_total"]
fallback_total = self._metrics["fallback_total"]
success_total = self._metrics["success_total"]
fail_total = self._metrics["fail_total"]
return {
**self._metrics,
"fallback_hit_rate": round(fallback_total / requests_total, 4) if requests_total else 0.0,
"success_rate": round(success_total / requests_total, 4) if requests_total else 0.0,
"failure_rate": round(fail_total / requests_total, 4) if requests_total else 0.0,
}

View File

@@ -0,0 +1,146 @@
from __future__ import annotations
import json
import time
from functools import lru_cache
from threading import Lock
from typing import Any, Protocol
from app.core.config import get_settings
from app.core.logging import logger
try:
from redis import Redis
except Exception: # pragma: no cover
Redis = None # type: ignore[assignment]
class Cache(Protocol):
def get(self, key: str): ...
def set(self, key: str, value: Any) -> None: ...
def delete(self, key: str) -> None: ...
def clear(self) -> None: ...
def stats(self) -> dict[str, int | float | str]: ...
class TTLCache:
def __init__(self, ttl_seconds: int):
self.ttl_seconds = ttl_seconds
self._store: dict[str, tuple[float, Any]] = {}
self._lock = Lock()
self._hits = 0
self._misses = 0
def get(self, key: str):
now = time.time()
with self._lock:
item = self._store.get(key)
if item is None:
self._misses += 1
return None
expires_at, value = item
if expires_at < now:
self._store.pop(key, None)
self._misses += 1
return None
self._hits += 1
return value
def set(self, key: str, value: Any) -> None:
expires_at = time.time() + self.ttl_seconds
with self._lock:
self._store[key] = (expires_at, value)
def delete(self, key: str) -> None:
with self._lock:
self._store.pop(key, None)
def clear(self) -> None:
with self._lock:
self._store.clear()
def stats(self) -> dict[str, int | float | str]:
with self._lock:
requests = self._hits + self._misses
hit_rate = (self._hits / requests) if requests else 0.0
return {
"backend": "memory",
"size": len(self._store),
"hits": self._hits,
"misses": self._misses,
"hit_rate": round(hit_rate, 4),
}
class RedisCache:
def __init__(self, url: str, prefix: str, ttl_seconds: int):
if Redis is None:
raise RuntimeError("redis package is not installed")
self.client = Redis.from_url(url, decode_responses=True)
self.prefix = prefix
self.ttl_seconds = ttl_seconds
self._hits = 0
self._misses = 0
self._lock = Lock()
def get(self, key: str):
raw = self.client.get(self._key(key))
with self._lock:
if raw is None:
self._misses += 1
return None
self._hits += 1
return json.loads(raw)
def set(self, key: str, value: Any) -> None:
self.client.set(self._key(key), json.dumps(value, ensure_ascii=False), ex=self.ttl_seconds)
def delete(self, key: str) -> None:
self.client.delete(self._key(key))
def clear(self) -> None:
pattern = f"{self.prefix}:*"
cursor = 0
while True:
cursor, keys = self.client.scan(cursor=cursor, match=pattern, count=200)
if keys:
self.client.delete(*keys)
if cursor == 0:
break
def stats(self) -> dict[str, int | float | str]:
with self._lock:
requests = self._hits + self._misses
hit_rate = (self._hits / requests) if requests else 0.0
return {
"backend": "redis",
"size": int(self.client.dbsize()),
"hits": self._hits,
"misses": self._misses,
"hit_rate": round(hit_rate, 4),
}
def _key(self, key: str) -> str:
return f"{self.prefix}:{key}"
def _build_cache(namespace: str, ttl_seconds: int) -> Cache:
settings = get_settings()
if settings.cache_backend == "redis":
try:
return RedisCache(settings.redis_url, f"{settings.redis_prefix}:{namespace}", ttl_seconds=ttl_seconds)
except Exception:
logger.exception("failed to init redis cache namespace=%s fallback to memory cache", namespace)
return TTLCache(ttl_seconds=ttl_seconds)
@lru_cache
def get_match_cache() -> Cache:
settings = get_settings()
return _build_cache("match", settings.match_cache_ttl_seconds)
@lru_cache
def get_query_cache() -> Cache:
settings = get_settings()
return _build_cache("query", settings.query_cache_ttl_seconds)

View File

@@ -26,13 +26,9 @@ class ExtractionService:
def extract_job(self, text: str) -> ExtractResponse:
logger.info("extract_job request text=%s", text)
llm_result = self._llm_extract(text, self.settings.prompt_dir / "job_extract.md")
if llm_result:
try:
return ExtractResponse(success=True, data=JobCard(**llm_result.content))
except ValidationError as exc:
logger.exception("LLM job extraction validation failed")
return ExtractResponse(success=False, errors=[str(exc)], missing_fields=self._missing_fields(exc))
llm_card = self._llm_extract_with_retry(text, self.settings.prompt_dir / "job_extract.md", JobCard)
if llm_card:
return ExtractResponse(success=True, data=llm_card)
try:
card = self._extract_job_rule(text)
@@ -43,13 +39,9 @@ class ExtractionService:
def extract_worker(self, text: str) -> ExtractResponse:
logger.info("extract_worker request text=%s", text)
llm_result = self._llm_extract(text, self.settings.prompt_dir / "worker_extract.md")
if llm_result:
try:
return ExtractResponse(success=True, data=WorkerCard(**llm_result.content))
except ValidationError as exc:
logger.exception("LLM worker extraction validation failed")
return ExtractResponse(success=False, errors=[str(exc)], missing_fields=self._missing_fields(exc))
llm_card = self._llm_extract_with_retry(text, self.settings.prompt_dir / "worker_extract.md", WorkerCard)
if llm_card:
return ExtractResponse(success=True, data=llm_card)
try:
card = self._extract_worker_rule(text)
@@ -65,6 +57,57 @@ class ExtractionService:
logger.exception("LLM extraction failed, fallback to rule-based extraction")
return None
def _llm_extract_with_retry(self, text: str, prompt_path: Path, schema_cls):
base_prompt = load_prompt(prompt_path)
llm_result = self._llm_extract(text, prompt_path)
if not llm_result:
return None
try:
return schema_cls(**llm_result.content)
except ValidationError as exc:
logger.warning("LLM extraction validation failed, trying schema-aware retry")
last_error = exc
last_output = llm_result.content
for _ in range(self.settings.extraction_llm_max_retries):
missing_fields = self._missing_fields(last_error)
repair_prompt = self._build_repair_prompt(base_prompt, schema_cls, missing_fields)
try:
repair_result = self.llm_client.extract_json(
repair_prompt,
self._build_repair_input(text, last_output, missing_fields),
)
except Exception:
logger.exception("LLM schema-aware retry failed")
return None
if not repair_result:
return None
last_output = repair_result.content
try:
return schema_cls(**repair_result.content)
except ValidationError as exc:
last_error = exc
logger.warning("LLM schema-aware retry still invalid missing_fields=%s", self._missing_fields(exc))
return None
def _build_repair_prompt(self, base_prompt: str, schema_cls, missing_fields: list[str]) -> str:
schema_json = json.dumps(schema_cls.model_json_schema(), ensure_ascii=False)
return (
f"{base_prompt}\n\n"
"你是结构化修复助手。请严格输出可被 JSON 解析的对象,不要输出解释文字。\n"
"目标是根据给定 schema 修复字段缺失和类型错误,优先保证必填字段完整。\n"
f"缺失或错误字段: {', '.join(missing_fields) if missing_fields else 'unknown'}\n"
f"JSON Schema: {schema_json}\n"
)
def _build_repair_input(self, original_text: str, last_output: dict, missing_fields: list[str]) -> str:
return (
f"原始文本:\n{original_text}\n\n"
f"上一次抽取结果:\n{json.dumps(last_output, ensure_ascii=False)}\n\n"
f"请重点修复字段:\n{json.dumps(missing_fields, ensure_ascii=False)}"
)
def _extract_job_rule(self, text: str) -> JobCard:
skill_hits = [item for item in self.skills if item in text]
category = next((item for item in self.categories if item in text), "活动执行")

View File

@@ -0,0 +1,105 @@
from __future__ import annotations
from dataclasses import dataclass
from queue import Empty, Full, Queue
from threading import Event, Lock, Thread
from typing import Any
from app.core.config import Settings
from app.core.logging import logger
from app.db.session import SessionLocal
from app.domain.schemas import JobCard, WorkerCard
from app.services.ingest_service import IngestService
from app.utils.ids import generate_id
@dataclass
class QueueTask:
task_id: str
kind: str
payload: dict[str, Any]
class IngestQueue:
def __init__(self, settings: Settings):
self.settings = settings
self.queue: Queue[QueueTask] = Queue(maxsize=settings.ingest_queue_max_size)
self._stop_event = Event()
self._thread: Thread | None = None
self._lock = Lock()
self._status: dict[str, str] = {}
self._processed = 0
self._failed = 0
def start(self) -> None:
if not self.settings.ingest_async_enabled:
return
if self._thread and self._thread.is_alive():
return
self._thread = Thread(target=self._run, daemon=True, name="ingest-queue-worker")
self._thread.start()
logger.info("ingest queue worker started")
def stop(self) -> None:
self._stop_event.set()
if self._thread and self._thread.is_alive():
self._thread.join(timeout=3)
def enqueue_job(self, card: JobCard) -> str:
return self._enqueue("job", card.model_dump(mode="json"))
def enqueue_worker(self, card: WorkerCard) -> str:
return self._enqueue("worker", card.model_dump(mode="json"))
def task_status(self, task_id: str) -> str:
with self._lock:
return self._status.get(task_id, "not_found")
def stats(self) -> dict[str, int]:
with self._lock:
return {
"queued": self.queue.qsize(),
"processed": self._processed,
"failed": self._failed,
}
def _enqueue(self, kind: str, payload: dict[str, Any]) -> str:
task_id = generate_id("queue")
task = QueueTask(task_id=task_id, kind=kind, payload=payload)
with self._lock:
self._status[task_id] = "queued"
try:
self.queue.put_nowait(task)
except Full as exc:
with self._lock:
self._status[task_id] = "rejected"
raise RuntimeError("ingest queue is full") from exc
return task_id
def _run(self) -> None:
while not self._stop_event.is_set():
try:
task = self.queue.get(timeout=0.5)
except Empty:
continue
try:
with self._lock:
self._status[task.task_id] = "processing"
with SessionLocal() as db:
service = IngestService(db)
if task.kind == "job":
service.ingest_job(JobCard(**task.payload))
elif task.kind == "worker":
service.ingest_worker(WorkerCard(**task.payload))
else:
raise ValueError(f"unknown task kind {task.kind}")
with self._lock:
self._status[task.task_id] = "done"
self._processed += 1
except Exception:
logger.exception("ingest queue task failed task_id=%s kind=%s", task.task_id, task.kind)
with self._lock:
self._status[task.task_id] = "failed"
self._failed += 1
finally:
self.queue.task_done()

View File

@@ -9,6 +9,7 @@ from app.core.logging import logger
from app.domain.schemas import BootstrapResponse, JobCard, WorkerCard
from app.repositories.job_repository import JobRepository
from app.repositories.worker_repository import WorkerRepository
from app.services.cache_service import get_match_cache, get_query_cache
from app.services.rag.lightrag_adapter import LightRAGAdapter
@@ -19,17 +20,27 @@ class IngestService:
self.job_repository = JobRepository(db)
self.worker_repository = WorkerRepository(db)
self.rag = LightRAGAdapter(self.settings)
self.match_cache = get_match_cache()
self.query_cache = get_query_cache()
def ingest_job(self, card: JobCard) -> JobCard:
logger.info("ingest_job job_id=%s", card.job_id)
self.job_repository.upsert(card)
self.rag.upsert_job(card)
if self.settings.match_cache_enabled:
self.match_cache.clear()
if self.settings.query_cache_enabled:
self.query_cache.clear()
return card
def ingest_worker(self, card: WorkerCard) -> WorkerCard:
logger.info("ingest_worker worker_id=%s", card.worker_id)
self.worker_repository.upsert(card)
self.rag.upsert_worker(card)
if self.settings.match_cache_enabled:
self.match_cache.clear()
if self.settings.query_cache_enabled:
self.query_cache.clear()
return card
def bootstrap(self) -> BootstrapResponse:
@@ -43,6 +54,10 @@ class IngestService:
self.ingest_job(JobCard(**item))
for item in workers:
self.ingest_worker(WorkerCard(**item))
if self.settings.match_cache_enabled:
self.match_cache.clear()
if self.settings.query_cache_enabled:
self.query_cache.clear()
return BootstrapResponse(
jobs=len(jobs),
workers=len(workers),

View File

@@ -6,14 +6,17 @@ import httpx
from app.core.config import Settings
from app.domain.schemas import PromptOutput
from app.services.ai_guard import AIGuard
class LLMClient:
def __init__(self, settings: Settings):
self.settings = settings
self.guard = AIGuard(settings)
def extract_json(self, system_prompt: str, user_text: str) -> PromptOutput | None:
if not self.settings.llm_enabled or not self.settings.llm_base_url or not self.settings.llm_api_key:
self.guard.record_fallback()
return None
payload = {
@@ -25,10 +28,77 @@ class LLMClient:
"temperature": 0.1,
"response_format": {"type": "json_object"},
}
headers = {"Authorization": f"Bearer {self.settings.llm_api_key}"}
with httpx.Client(timeout=30.0) as client:
response = client.post(f"{self.settings.llm_base_url.rstrip('/')}/chat/completions", json=payload, headers=headers)
response.raise_for_status()
data = response.json()
raw_text = data["choices"][0]["message"]["content"]
endpoints = [self.settings.llm_base_url, *self.settings.llm_fallback_base_urls]
raw_text = self._request_with_failover(
endpoints=endpoints,
path="/chat/completions",
payload=payload,
api_key=self.settings.llm_api_key,
)
if raw_text is None:
self.guard.record_fallback()
return None
return PromptOutput(content=json.loads(raw_text), raw_text=raw_text)
def embedding(self, text: str) -> list[float] | None:
if not self.settings.embedding_enabled:
return None
base_url = self.settings.embedding_base_url or self.settings.llm_base_url
api_key = self.settings.embedding_api_key or self.settings.llm_api_key
if not base_url or not api_key:
self.guard.record_fallback()
return None
payload = {
"model": self.settings.embedding_model,
"input": text,
}
endpoints = [base_url, *self.settings.embedding_fallback_base_urls]
data = self._request_with_failover(
endpoints=endpoints,
path="/embeddings",
payload=payload,
api_key=api_key,
return_full_response=True,
)
if data is None:
self.guard.record_fallback()
return None
embedding = data["data"][0]["embedding"]
if not isinstance(embedding, list):
return None
return [float(item) for item in embedding]
def metrics(self) -> dict:
return self.guard.snapshot()
def _request_with_failover(
self,
endpoints: list[str],
path: str,
payload: dict,
api_key: str,
return_full_response: bool = False,
):
if not endpoints:
return None
for index, endpoint in enumerate([item for item in endpoints if item]):
allowed, _ = self.guard.allow_request(endpoint)
if not allowed:
continue
if index > 0:
self.guard.record_failover()
try:
headers = {"Authorization": f"Bearer {api_key}"}
with httpx.Client(timeout=self.settings.ai_request_timeout_seconds) as client:
response = client.post(f"{endpoint.rstrip('/')}{path}", json=payload, headers=headers)
response.raise_for_status()
data = response.json()
self.guard.record_success(endpoint)
if return_full_response:
return data
return data["choices"][0]["message"]["content"]
except Exception:
self.guard.record_failure(endpoint)
continue
return None

View File

@@ -0,0 +1,121 @@
from __future__ import annotations
from dataclasses import dataclass
from queue import Empty, Full, Queue
from threading import Event, Lock, Thread
from typing import Any
from app.core.config import Settings
from app.core.logging import logger
from app.db.session import SessionLocal
from app.domain.schemas import MatchResult
from app.repositories.job_repository import JobRepository
from app.repositories.worker_repository import WorkerRepository
from app.services.card_mapper import job_to_card, worker_to_card
from app.services.matching_service import MatchingService
from app.utils.ids import generate_id
@dataclass
class MatchTask:
task_id: str
kind: str
source_id: str
top_n: int
class MatchQueue:
def __init__(self, settings: Settings):
self.settings = settings
self.queue: Queue[MatchTask] = Queue(maxsize=settings.match_queue_max_size)
self._stop_event = Event()
self._thread: Thread | None = None
self._lock = Lock()
self._status: dict[str, str] = {}
self._results: dict[str, list[dict[str, Any]]] = {}
self._processed = 0
self._failed = 0
def start(self) -> None:
if not self.settings.match_async_enabled:
return
if self._thread and self._thread.is_alive():
return
self._thread = Thread(target=self._run, daemon=True, name="match-queue-worker")
self._thread.start()
logger.info("match queue worker started")
def stop(self) -> None:
self._stop_event.set()
if self._thread and self._thread.is_alive():
self._thread.join(timeout=3)
def enqueue_workers(self, job_id: str, top_n: int) -> str:
return self._enqueue("workers", job_id, top_n)
def enqueue_jobs(self, worker_id: str, top_n: int) -> str:
return self._enqueue("jobs", worker_id, top_n)
def task_status(self, task_id: str) -> str:
with self._lock:
return self._status.get(task_id, "not_found")
def task_result(self, task_id: str) -> list[dict[str, Any]] | None:
with self._lock:
return self._results.get(task_id)
def stats(self) -> dict[str, int]:
with self._lock:
return {
"queued": self.queue.qsize(),
"processed": self._processed,
"failed": self._failed,
}
def _enqueue(self, kind: str, source_id: str, top_n: int) -> str:
task_id = generate_id("mq")
task = MatchTask(task_id=task_id, kind=kind, source_id=source_id, top_n=top_n)
with self._lock:
self._status[task_id] = "queued"
try:
self.queue.put_nowait(task)
except Full as exc:
with self._lock:
self._status[task_id] = "rejected"
raise RuntimeError("match queue is full") from exc
return task_id
def _run(self) -> None:
while not self._stop_event.is_set():
try:
task = self.queue.get(timeout=0.5)
except Empty:
continue
try:
with self._lock:
self._status[task.task_id] = "processing"
with SessionLocal() as db:
service = MatchingService(db)
if task.kind == "workers":
job = JobRepository(db).get(task.source_id)
if job is None:
raise ValueError("job not found")
items = service.match_workers(job_to_card(job), task.top_n)
elif task.kind == "jobs":
worker = WorkerRepository(db).get(task.source_id)
if worker is None:
raise ValueError("worker not found")
items = service.match_jobs(worker_to_card(worker), task.top_n)
else:
raise ValueError(f"unknown task kind {task.kind}")
with self._lock:
self._status[task.task_id] = "done"
self._results[task.task_id] = [item.model_dump(mode="json") for item in items]
self._processed += 1
except Exception:
logger.exception("match queue task failed task_id=%s kind=%s", task.task_id, task.kind)
with self._lock:
self._status[task.task_id] = "failed"
self._failed += 1
finally:
self.queue.task_done()

View File

@@ -10,8 +10,10 @@ from app.domain.schemas import JobCard, MatchBreakdown, MatchResult, QueryFilter
from app.repositories.job_repository import JobRepository
from app.repositories.match_repository import MatchRepository
from app.repositories.worker_repository import WorkerRepository
from app.services.cache_service import get_match_cache
from app.services.card_mapper import job_to_card, worker_to_card
from app.services.rag.lightrag_adapter import LightRAGAdapter
from app.services.weight_service import MatchWeightService
from app.utils.ids import generate_id
@@ -23,9 +25,16 @@ class MatchingService:
self.workers = WorkerRepository(db)
self.matches = MatchRepository(db)
self.rag = LightRAGAdapter(self.settings)
self.weight_service = MatchWeightService(self.settings)
self.cache = get_match_cache()
def match_workers(self, source: JobCard, top_n: int) -> list[MatchResult]:
logger.info("match_workers source_id=%s top_n=%s", source.job_id, top_n)
cache_key = f"match_workers:{source.job_id}:{top_n}"
if self.settings.match_cache_enabled:
cached = self.cache.get(cache_key)
if cached is not None:
return self._parse_cached_matches(cached)
query_text = " ".join([source.title, source.category, source.city, source.region, *source.skills, *source.tags])
candidate_ids = self.rag.search(
query_text=query_text,
@@ -36,10 +45,17 @@ class MatchingService:
results = [self._build_job_to_worker_match(source, worker_to_card(worker)) for worker in candidates]
results = sorted(results, key=lambda item: item.match_score, reverse=True)[:top_n]
self.matches.bulk_replace(results, SourceType.job_to_worker.value, source.job_id)
if self.settings.match_cache_enabled:
self.cache.set(cache_key, [item.model_dump(mode="json") for item in results])
return results
def match_jobs(self, source: WorkerCard, top_n: int) -> list[MatchResult]:
logger.info("match_jobs source_id=%s top_n=%s", source.worker_id, top_n)
cache_key = f"match_jobs:{source.worker_id}:{top_n}"
if self.settings.match_cache_enabled:
cached = self.cache.get(cache_key)
if cached is not None:
return self._parse_cached_matches(cached)
query_text = " ".join([source.name, *source.cities, *source.regions, *[item.name for item in source.skills], *source.experience_tags])
city = source.cities[0] if source.cities else None
candidate_ids = self.rag.search(
@@ -51,6 +67,8 @@ class MatchingService:
results = [self._build_worker_to_job_match(source, job_to_card(job)) for job in candidates]
results = sorted(results, key=lambda item: item.match_score, reverse=True)[:top_n]
self.matches.bulk_replace(results, SourceType.worker_to_job.value, source.worker_id)
if self.settings.match_cache_enabled:
self.cache.set(cache_key, [item.model_dump(mode="json") for item in results])
return results
def explain(self, match_id: str) -> MatchResult | None:
@@ -61,6 +79,20 @@ class MatchingService:
return match_record_to_schema(record)
def feedback(self, match_id: str, accepted: bool) -> dict[str, float] | None:
record = self.matches.get(match_id)
if record is None:
return None
from app.services.card_mapper import match_record_to_schema
match = match_record_to_schema(record)
if self.settings.ranking_learning_enabled:
return self.weight_service.update_from_feedback(match.breakdown, accepted)
return self.weight_service.get_weights()
def current_weights(self) -> dict[str, float]:
return self.weight_service.get_weights()
def _build_job_to_worker_match(self, job: JobCard, worker: WorkerCard) -> MatchResult:
job_skills = set(job.skills)
expanded_skills = self.rag.expand_skills(job.skills)
@@ -143,13 +175,14 @@ class MatchingService:
experience_score: float,
reliability_score: float,
) -> float:
return (
self.settings.score_skill_weight * skill_score
+ self.settings.score_region_weight * region_score
+ self.settings.score_time_weight * time_score
+ self.settings.score_experience_weight * experience_score
+ self.settings.score_reliability_weight * reliability_score
breakdown = MatchBreakdown(
skill_score=skill_score,
region_score=region_score,
time_score=time_score,
experience_score=experience_score,
reliability_score=reliability_score,
)
return self.weight_service.score(breakdown)
def _build_reasons(
self,
@@ -176,3 +209,10 @@ class MatchingService:
while len(reasons) < 3:
reasons.append("岗位需求与候选画像存在基础匹配")
return reasons[:5]
def _parse_cached_matches(self, cached) -> list[MatchResult]:
if isinstance(cached, list) and cached and isinstance(cached[0], MatchResult):
return cached
if isinstance(cached, list):
return [MatchResult(**item) for item in cached]
return []

View File

@@ -10,6 +10,7 @@ from qdrant_client import QdrantClient, models
from app.core.config import Settings
from app.core.logging import logger
from app.domain.schemas import JobCard, QueryFilters, WorkerCard
from app.services.llm_client import LLMClient
class LightRAGAdapter:
@@ -17,13 +18,28 @@ class LightRAGAdapter:
self.settings = settings
self.client = QdrantClient(url=settings.qdrant_url)
self.skill_graph = self._load_skill_graph()
self.llm_client = LLMClient(settings)
self.collection_vector_size: int | None = None
def ensure_ready(self) -> None:
collections = {item.name for item in self.client.get_collections().collections}
expected_size = self._configured_vector_size()
if self.settings.qdrant_collection not in collections:
self.client.create_collection(
collection_name=self.settings.qdrant_collection,
vectors_config=models.VectorParams(size=self.settings.vector_size, distance=models.Distance.COSINE),
vectors_config=models.VectorParams(size=expected_size, distance=models.Distance.COSINE),
)
self.collection_vector_size = expected_size
return
info = self.client.get_collection(self.settings.qdrant_collection)
configured_size = info.config.params.vectors.size
self.collection_vector_size = int(configured_size)
if self.collection_vector_size != expected_size:
logger.warning(
"qdrant vector size mismatch, collection=%s expected=%s actual=%s; using actual size",
self.settings.qdrant_collection,
expected_size,
self.collection_vector_size,
)
def health(self) -> str:
@@ -125,14 +141,40 @@ class LightRAGAdapter:
)
def _vectorize(self, text: str) -> list[float]:
vector = [0.0 for _ in range(self.settings.vector_size)]
if self.settings.embedding_enabled and self.settings.embedding_backend == "openai_compatible":
try:
embedding = self.llm_client.embedding(text)
if embedding:
return self._normalize_embedding(embedding)
except Exception:
logger.exception("embedding request failed, fallback to hash vector")
target_size = self._active_vector_size()
vector = [0.0 for _ in range(target_size)]
tokens = self._tokenize(text)
for token in tokens:
index = hash(token) % self.settings.vector_size
index = hash(token) % target_size
vector[index] += 1.0
norm = math.sqrt(sum(item * item for item in vector)) or 1.0
return [item / norm for item in vector]
def _normalize_embedding(self, embedding: list[float]) -> list[float]:
target_size = self._active_vector_size()
vector = embedding[:target_size]
if len(vector) < target_size:
vector.extend([0.0] * (target_size - len(vector)))
norm = math.sqrt(sum(item * item for item in vector)) or 1.0
return [item / norm for item in vector]
def _active_vector_size(self) -> int:
if self.collection_vector_size:
return self.collection_vector_size
return self._configured_vector_size()
def _configured_vector_size(self) -> int:
if self.settings.embedding_enabled and self.settings.embedding_backend == "openai_compatible":
return self.settings.embedding_vector_size
return self.settings.vector_size
def _tokenize(self, text: str) -> list[str]:
cleaned = [part.strip().lower() for part in text.replace("", " ").replace("", " ").replace("", " ").split()]
tokens = [part for part in cleaned if part]

View File

@@ -0,0 +1,23 @@
from __future__ import annotations
from functools import lru_cache
from app.core.config import get_settings
from app.services.ingest_queue import IngestQueue
from app.services.match_queue import MatchQueue
from app.services.traffic_guard import TrafficGuard
@lru_cache
def get_ingest_queue() -> IngestQueue:
return IngestQueue(get_settings())
@lru_cache
def get_match_queue() -> MatchQueue:
return MatchQueue(get_settings())
@lru_cache
def get_traffic_guard() -> TrafficGuard:
return TrafficGuard(get_settings())

View File

@@ -0,0 +1,108 @@
from __future__ import annotations
import time
from collections import deque
from threading import Lock
import httpx
from app.core.config import Settings
from app.core.logging import logger
class TrafficGuard:
def __init__(self, settings: Settings):
self.settings = settings
self._lock = Lock()
self._minute = 0
self._minute_count = 0
self._open_until = 0.0
self._events: deque[tuple[float, int]] = deque()
self._requests = 0
self._rate_limited = 0
self._circuit_blocked = 0
self._avg_latency_ms = 0.0
self._alert_last_sent = 0.0
def allow(self, path: str) -> tuple[bool, str]:
now = time.time()
with self._lock:
minute = int(now // 60)
if self._minute != minute:
self._minute = minute
self._minute_count = 0
if self._minute_count >= self.settings.app_rate_limit_per_minute:
self._rate_limited += 1
return False, "rate_limited"
if self._open_until > now and not self._is_exempt(path):
self._circuit_blocked += 1
return False, "circuit_open"
self._minute_count += 1
self._requests += 1
return True, "ok"
def record(self, status_code: int, latency_ms: float) -> None:
now = time.time()
with self._lock:
self._events.append((now, status_code))
self._avg_latency_ms = self._ema(self._avg_latency_ms, latency_ms)
self._trim(now)
total = len(self._events)
if total < self.settings.app_circuit_breaker_min_requests:
return
errors = sum(1 for _, code in self._events if code >= 500)
error_rate = errors / total
if error_rate >= self.settings.app_circuit_breaker_error_rate:
self._open_until = now + self.settings.app_circuit_breaker_cooldown_seconds
self._send_alert(
"app circuit opened",
{
"error_rate": round(error_rate, 4),
"window_requests": total,
"cooldown_seconds": self.settings.app_circuit_breaker_cooldown_seconds,
},
)
def snapshot(self) -> dict[str, float | int]:
now = time.time()
with self._lock:
self._trim(now)
total = len(self._events)
errors = sum(1 for _, code in self._events if code >= 500)
return {
"requests_total": self._requests,
"rate_limited_total": self._rate_limited,
"circuit_blocked_total": self._circuit_blocked,
"window_requests": total,
"window_errors": errors,
"window_error_rate": round((errors / total), 4) if total else 0.0,
"avg_latency_ms": round(self._avg_latency_ms, 2),
"circuit_open": 1 if self._open_until > now else 0,
}
def _trim(self, now: float) -> None:
lower = now - self.settings.app_circuit_breaker_window_seconds
while self._events and self._events[0][0] < lower:
self._events.popleft()
def _ema(self, prev: float, value: float, alpha: float = 0.2) -> float:
if prev <= 0:
return value
return alpha * value + (1 - alpha) * prev
def _is_exempt(self, path: str) -> bool:
return path in {"/health", "/docs", "/openapi.json", "/poc/ops/system/metrics", "/poc/ops/ai/metrics"}
def _send_alert(self, message: str, extra: dict) -> None:
now = time.time()
if now - self._alert_last_sent < 30:
return
self._alert_last_sent = now
logger.warning("%s extra=%s", message, extra)
if not self.settings.alert_webhook_url:
return
try:
with httpx.Client(timeout=2.0) as client:
client.post(self.settings.alert_webhook_url, json={"message": message, "extra": extra})
except Exception:
logger.exception("alert webhook send failed")

View File

@@ -0,0 +1,77 @@
from __future__ import annotations
import json
from pathlib import Path
from app.core.config import Settings
from app.core.logging import logger
from app.domain.schemas import MatchBreakdown
class MatchWeightService:
def __init__(self, settings: Settings):
self.settings = settings
self.path: Path = settings.match_weights_path
def default_weights(self) -> dict[str, float]:
return {
"skill": self.settings.score_skill_weight,
"region": self.settings.score_region_weight,
"time": self.settings.score_time_weight,
"experience": self.settings.score_experience_weight,
"reliability": self.settings.score_reliability_weight,
}
def get_weights(self) -> dict[str, float]:
weights = self.default_weights()
if not self.path.exists():
return self._normalize(weights)
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
for key in weights:
value = data.get(key)
if isinstance(value, (int, float)):
weights[key] = float(value)
except Exception:
logger.exception("failed to read learned ranking weights, fallback to defaults")
return self._normalize(weights)
def score(self, breakdown: MatchBreakdown) -> float:
weights = self.get_weights()
return (
weights["skill"] * breakdown.skill_score
+ weights["region"] * breakdown.region_score
+ weights["time"] * breakdown.time_score
+ weights["experience"] * breakdown.experience_score
+ weights["reliability"] * breakdown.reliability_score
)
def update_from_feedback(self, breakdown: MatchBreakdown, accepted: bool) -> dict[str, float]:
weights = self.get_weights()
features = {
"skill": breakdown.skill_score,
"region": breakdown.region_score,
"time": breakdown.time_score,
"experience": breakdown.experience_score,
"reliability": breakdown.reliability_score,
}
target = 1.0 if accepted else 0.0
prediction = sum(weights[name] * value for name, value in features.items())
error = target - prediction
lr = self.settings.ranking_learning_rate
updated = {name: max(0.0, weights[name] + lr * error * value) for name, value in features.items()}
normalized = self._normalize(updated)
self._save_weights(normalized)
return normalized
def _save_weights(self, weights: dict[str, float]) -> None:
self.settings.data_dir.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(weights, ensure_ascii=False, indent=2), encoding="utf-8")
def _normalize(self, weights: dict[str, float]) -> dict[str, float]:
total = sum(max(value, 0.0) for value in weights.values())
if total <= 0:
fallback = self.default_weights()
total = sum(fallback.values())
return {key: value / total for key, value in fallback.items()}
return {key: max(value, 0.0) / total for key, value in weights.items()}