M4 收口:神识用户面与计量闭环——BYOK/配额三段式/usage 落库/Anthropic+Gemini 适配器/神识阁

- 神识 Go:store 双实现(PG 自动建表+60s 就绪重试 / 内存回退)承载 usage_logs·byok_keys·quota_balances;BYOK 密钥库(AES-GCM 加密、API 脱敏、撤回即失效、主密钥未配置 fail-closed);配额三段式(预扣 max_tokens 钳位 1M→结算实际 token→全败幂等退款,HJ-2004);计量归属(IR user_id→usage_logs,/me/usage 用户维度聚合+配额总览,/admin/usage/sample 对账抽样);Anthropic/Gemini 出站适配器(SSE 解析、system 提升、用量映射);stub 端点补名修复计量空 endpoint
- IR 契约:user_id/byok_key_id 双端同步(Go types.go + Py types.py)
- 灵台:runs.owner_id(迁移 005,server_default 兼容存量)经 kernel 全链透传为 IR user_id;shenshi_client 扩展用户面方法(X-Hunjing-User-Id 代理,Echo 桩带内存实现);/me/byok-keys 登记·列表·撤回与 /me/usage 端点(4 用例)
- 前端神识阁(/models 重建):偏好(生成参数表单)/自带密钥(登记加密保存、脱敏列表、撤回二次确认)/用量(聚合表+配额进度条)三区;me API 库+mock+5 用例;侧栏恢复入口
- 配置与部署:.env.example 增 SHENSHI_DATABASE_URL/SHENSHI_BYOK_MASTER_KEY/SHENSHI_USER_BUDGET_TOKENS;compose 神识容器接自有库(init-shenshi.sql 预建 hunjing_shenshi,与灵台不共库共表)
- 证据:Go 34 用例(BYOK 独占/配额拒绝/结算退款/归属聚合)、后端 532、前端 210、BFF 4 全绿;跨进程 E2E 四场景(灵台 HttpShenshiClient→真 Go 神识:登记脱敏/BYOK 独占不可达整体失败/用量归属聚合/撤回即失效)通过;rewrite-plan M4 标记完成、能力矩阵神识行升已验证
This commit is contained in:
chenxin committed 2026-08-27 18:39:17 +08:00
1 parent 462d570751
commit 144415247e
51 files changed
+3533 -90

No files matched your search

@@ -0,0 +1,40 @@
"""运行归属用户列_runs_owner_id_用量计量透传神识
Revision ID: 005
Revises: 004
Create Date: 2026-08-27 10:31:10.225717
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '005'
down_revision: Union[str, Sequence[str], None] = '004'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('ix_factual_records_canonization_id'), 'factual_records', ['canonization_id'], unique=False)
op.create_index(op.f('ix_factual_records_character_id'), 'factual_records', ['character_id'], unique=False)
op.create_index(op.f('ix_memory_triggers_memory_id'), 'memory_triggers', ['memory_id'], unique=False)
op.add_column('runs', sa.Column('owner_id', sa.String(length=80), server_default='', nullable=False))
op.create_index(op.f('ix_runs_owner_id'), 'runs', ['owner_id'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_runs_owner_id'), table_name='runs')
op.drop_column('runs', 'owner_id')
op.drop_index(op.f('ix_memory_triggers_memory_id'), table_name='memory_triggers')
op.drop_index(op.f('ix_factual_records_character_id'), table_name='factual_records')
op.drop_index(op.f('ix_factual_records_canonization_id'), table_name='factual_records')
# ### end Alembic commands ###
+2
View File
@@ -17,6 +17,7 @@ from app.api.v1.endpoints import (
jingsheng_characters,
kernel_runs,
lorebooks,
me,
mirror_trees,
mcp,
personality,
@@ -37,6 +38,7 @@ api_router.include_router(characters.router)
api_router.include_router(character_import.router)
api_router.include_router(runs.router)
api_router.include_router(kernel_runs.router)
api_router.include_router(me.router)
api_router.include_router(embryo_drafts.router)
api_router.include_router(jingsheng_characters.router)
api_router.include_router(mirror_trees.router)
@@ -111,7 +111,8 @@ async def rehearse_embryo(
f"身份锚点:{json.dumps(baseline, ensure_ascii=False)}"
)
run = await kernel.start_rehearsal_run(
embryo_id=embryo_id, prompt=payload.prompt, system_contract=system_contract
embryo_id=embryo_id, prompt=payload.prompt, system_contract=system_contract,
owner_id=user.owner_scope,
)
await jingsheng.record_rehearsal(
embryo_id, user.owner_scope, run_id=run.id, prompt=payload.prompt
+1 -1
View File
@@ -50,6 +50,6 @@ async def create_kernel_run(
db, payload.character_id, owner_scope=user.owner_scope
)
run = await kernel.start_foreground_run(
character_id=character.id, prompt=payload.prompt
character_id=character.id, prompt=payload.prompt, owner_id=user.owner_scope
)
return await get_run_response(run.id)
+71
View File
@@ -0,0 +1,71 @@
"""用户面 API(神识阁数据源):BYOK 密钥管理与用量查询。
灵台是神识唯一调用方:本端点鉴权后以 ``X-Hunjing-User-Id`` 代理转发;
密钥明文只进不出(登记请求体 → 神识加密落库,响应永远脱敏)。
"""
from fastapi import APIRouter, Depends, Request, status
from app.api.v1.deps import get_current_user
from app.core.errors import AppError
from app.kernel import Kernel
from app.schemas.auth import CurrentUser
from app.schemas.me import (
ByokKeyListResponse,
MyUsageResponse,
RegisterByokKeyRequest,
)
from app.shenshi_client.client import ShenshiClient
from app.shenshi_client.types import ShenshiError
router = APIRouter(prefix="/me", tags=["me"])
def _shenshi(request: Request) -> ShenshiClient:
kernel: Kernel | None = getattr(request.app.state, "kernel", None)
if kernel is None:
raise AppError("service_unavailable", "Kernel 未就绪。", status_code=503)
return kernel.shenshi
@router.get("/byok-keys", response_model=ByokKeyListResponse)
async def list_byok_keys(
request: Request, user: CurrentUser = Depends(get_current_user)
) -> ByokKeyListResponse:
keys = await _shenshi(request).list_byok_keys(user.owner_scope)
return ByokKeyListResponse(keys=keys)
@router.post("/byok-keys", status_code=status.HTTP_201_CREATED)
async def register_byok_key(
payload: RegisterByokKeyRequest,
request: Request,
user: CurrentUser = Depends(get_current_user),
) -> dict:
key = await _shenshi(request).register_byok_key(
user.owner_scope,
name=payload.name,
type=payload.type,
base_url=payload.base_url,
api_key=payload.api_key,
model=payload.model,
)
return key
@router.delete("/byok-keys/{key_id}", status_code=status.HTTP_204_NO_CONTENT)
async def revoke_byok_key(
key_id: str, request: Request, user: CurrentUser = Depends(get_current_user)
) -> None:
try:
await _shenshi(request).revoke_byok_key(user.owner_scope, key_id)
except ShenshiError as exc:
raise AppError(exc.code, exc.message, status_code=404) from None
@router.get("/usage", response_model=MyUsageResponse)
async def my_usage(
request: Request, user: CurrentUser = Depends(get_current_user)
) -> MyUsageResponse:
data = await _shenshi(request).get_my_usage(user.owner_scope)
return MyUsageResponse(rows=data.get("rows", []), quota=data.get("quota"))
@@ -521,6 +521,7 @@ async def spawn_answer(
run = await kernel.start_foreground_run(
character_id=tree.character_id,
prompt=prompt,
owner_id=owner_scope,
system_contract=None,
origin="tree_answer",
on_run_final=_make_finalize(tree.id, tree.character_id, node_id),
@@ -620,6 +621,7 @@ async def _spawn_continuation(
run = await kernel.start_foreground_run(
character_id=tree.character_id,
prompt=prompt,
owner_id=owner_scope,
origin=f"tree_{kind}",
on_run_final=_make_finalize(tree.id, tree.character_id, node_id),
)
+8 -1
View File
@@ -51,6 +51,7 @@ class Kernel:
*,
character_id: str,
prompt: str,
owner_id: str = "",
trace_id: str | None = None,
system_contract: str | None = None,
origin: str = "kernel",
@@ -63,6 +64,7 @@ class Kernel:
async with self._sessions() as session:
run = Run(
character_id=character_id,
owner_id=owner_id,
status="queued",
kind="foreground",
goal=prompt[:300] if prompt else None,
@@ -75,6 +77,7 @@ class Kernel:
self._spawn(
run_id=run_id,
character_id=character_id,
owner_id=owner_id,
prompt=prompt,
trace_id=trace_id,
seat_key=None,
@@ -89,13 +92,14 @@ class Kernel:
return created
async def start_rehearsal_run(
self, *, embryo_id: str, prompt: str, system_contract: str
self, *, embryo_id: str, prompt: str, system_contract: str, owner_id: str = ""
) -> Run:
"""照心试演(幻境 Run):无正式人物、不占人物命台、不产生人物经历。"""
profile = resolve_profile()
async with self._sessions() as session:
run = Run(
character_id=None,
owner_id=owner_id,
status="queued",
kind="rehearsal",
goal=prompt[:300] if prompt else None,
@@ -108,6 +112,7 @@ class Kernel:
self._spawn(
run_id=run_id,
character_id=None,
owner_id=owner_id,
prompt=prompt,
trace_id=None,
seat_key=f"embryo:{embryo_id}",
@@ -129,6 +134,7 @@ class Kernel:
*,
run_id: str,
character_id: str | None,
owner_id: str,
prompt: str,
trace_id: str | None,
seat_key: str | None,
@@ -141,6 +147,7 @@ class Kernel:
self.loop.execute(
run_id=run_id,
character_id=character_id,
owner_id=owner_id,
prompt=prompt,
trace_id=trace_id,
seat_key=seat_key,
+6
View File
@@ -83,6 +83,7 @@ class KernelRunLoop:
run_id: str,
character_id: str | None,
prompt: str,
owner_id: str = "",
trace_id: str | None = None,
seat_key: str | None = None,
system_contract: str | None = None,
@@ -98,6 +99,7 @@ class KernelRunLoop:
await self._execute(
run_id=run_id,
character_id=character_id,
owner_id=owner_id,
prompt=prompt,
trace_id=trace,
seat_key=seat_key,
@@ -127,6 +129,7 @@ class KernelRunLoop:
*,
run_id: str,
character_id: str | None,
owner_id: str,
prompt: str,
trace_id: str,
seat_key: str | None,
@@ -153,6 +156,7 @@ class KernelRunLoop:
await self._turn(
run_id,
character_id,
owner_id,
trace_id,
prompt,
system_contract or _SYSTEM_CONTRACT_PLACEHOLDER,
@@ -165,6 +169,7 @@ class KernelRunLoop:
self,
run_id: str,
character_id: str | None,
owner_id: str,
trace_id: str,
prompt: str,
system_contract: str,
@@ -189,6 +194,7 @@ class KernelRunLoop:
run_id=run_id,
trace_id=trace_id,
character_id=character_id,
user_id=owner_id or None,
temperature=temperature,
max_tokens=max_tokens,
)
+2
View File
@@ -19,6 +19,8 @@ class Run(Base):
character_id: Mapped[str | None] = mapped_column(
String(80), ForeignKey("characters.id"), index=True, nullable=True
)
# Run 归属用户(用量计量与配额裁决维度,透传神识 user_id;空 = 平台调用)
owner_id: Mapped[str] = mapped_column(String(80), nullable=False, default="", index=True)
status: Mapped[str] = mapped_column(String(40), nullable=False, default="queued", index=True)
# run.kind:foreground / background_task / awakening / rehearsal(契约词典 §8.2)
kind: Mapped[str] = mapped_column(
+36
View File
@@ -0,0 +1,36 @@
"""用户面 API schema(神识阁数据源:BYOK 密钥与用量)。"""
from pydantic import BaseModel, Field
ByokType = str # openai_compatible / anthropic / gemini(契约词典 §14.2,神识侧校验)
class ByokKeyView(BaseModel):
"""BYOK 密钥脱敏形态(API 永不回明文)。"""
model_config = {"extra": "allow"} # 神识侧字段直通(id/name/type/base_url/model/api_key_mask/…)
class ByokKeyListResponse(BaseModel):
keys: list[dict] = Field(default_factory=list, description="脱敏密钥清单。")
class RegisterByokKeyRequest(BaseModel):
"""登记用户自带密钥。"""
name: str = Field(min_length=1, max_length=120, description="密钥备注名。")
type: ByokType = Field(description="端点类型:openai_compatible / anthropic / gemini。")
base_url: str = Field(min_length=1, max_length=500, description="Provider Base URL。")
api_key: str = Field(min_length=8, max_length=500, description="密钥明文(仅本次请求可见,落库即加密)。")
model: str = Field(min_length=1, max_length=200, description="该密钥承载的上游模型名。")
class UsageRow(BaseModel):
"""用量聚合行(用户×模型×端点)。"""
model_config = {"extra": "allow"}
class MyUsageResponse(BaseModel):
rows: list[dict] = Field(default_factory=list, description="用量聚合行。")
quota: dict | None = Field(default=None, description="配额总览(budget/used/reserved)。")
+101 -3
View File
@@ -11,7 +11,7 @@ from __future__ import annotations
import json
from collections.abc import AsyncIterator
from typing import Protocol
from typing import Any, Protocol
import httpx
@@ -19,19 +19,34 @@ from app.shenshi_client.types import ShenshiChatRequest, ShenshiDelta, ShenshiEr
class ShenshiClient(Protocol):
"""神识数据面契约:流式对话补全。"""
"""神识数据面契约:流式对话补全 + 用户面(BYOK/用量,神识阁数据源)。"""
def stream_chat(self, request: ShenshiChatRequest) -> AsyncIterator[ShenshiDelta]: ...
async def get_my_usage(self, user_id: str) -> dict[str, Any]: ...
async def list_byok_keys(self, user_id: str) -> list[dict[str, Any]]: ...
async def register_byok_key(
self, user_id: str, *, name: str, type: str, base_url: str, api_key: str, model: str
) -> dict[str, Any]: ...
async def revoke_byok_key(self, user_id: str, key_id: str) -> None: ...
class EchoShenshi:
"""进程内回声桩:把最后一条用户消息按固定粒度切分回流。
只用于 M0 骨架验证(登录 → 占位 Run → SSE 端到端),不模拟任何 Provider 行为。
数据面只用于骨架验证(登录 → 占位 Run → SSE 端到端);用户面为最小内存实现,
供神识阁在无神识部署时演示与测试桩。
"""
chunk_size = 8
def __init__(self) -> None:
self._byok: dict[str, list[dict[str, Any]]] = {}
self._usage: dict[str, list[dict[str, Any]]] = {}
async def stream_chat(self, request: ShenshiChatRequest) -> AsyncIterator[ShenshiDelta]:
prompt = next((m.content for m in reversed(request.messages) if m.role == "user"), "")
text = f"【回声桩】{prompt}"
@@ -42,6 +57,41 @@ class EchoShenshi:
usage_prompt_tokens=max(len(prompt) // 4, 1),
usage_completion_tokens=max(len(text) // 4, 1),
)
if request.user_id:
self._usage.setdefault(request.user_id, []).append(
{
"user_id": request.user_id,
"model": request.model or "",
"endpoint": "echo",
"requests": 1,
"ok": 1,
"prompt_tokens": max(len(prompt) // 4, 1),
"completion_tokens": max(len(text) // 4, 1),
}
)
async def get_my_usage(self, user_id: str) -> dict[str, Any]:
return {"rows": self._usage.get(user_id, [])}
async def list_byok_keys(self, user_id: str) -> list[dict[str, Any]]:
return self._byok.get(user_id, [])
async def register_byok_key(
self, user_id: str, *, name: str, type: str, base_url: str, api_key: str, model: str
) -> dict[str, Any]:
entry = {
"id": f"byok_echo_{len(self._byok.get(user_id, [])) + 1}",
"name": name,
"type": type,
"base_url": base_url,
"model": model,
"api_key_mask": api_key[:3] + "****",
}
self._byok.setdefault(user_id, []).append(entry)
return entry
async def revoke_byok_key(self, user_id: str, key_id: str) -> None:
self._byok[user_id] = [k for k in self._byok.get(user_id, []) if k["id"] != key_id]
class HttpShenshiClient:
@@ -56,6 +106,54 @@ class HttpShenshiClient:
self._headers = {"Authorization": f"Bearer {service_token}"} if service_token else {}
self._timeout = timeout
def _user_headers(self, user_id: str) -> dict[str, str]:
return {**self._headers, "X-Hunjing-User-Id": user_id}
async def _request_json(
self, method: str, path: str, *, user_id: str, json_body: dict[str, Any] | None = None
) -> Any:
async with httpx.AsyncClient(trust_env=False, timeout=self._timeout) as client:
response = await client.request(
method,
f"{self._base_url}{path}",
json=json_body,
headers=self._user_headers(user_id),
)
if response.status_code >= 400:
code = f"HJ-2{max(response.status_code - 400, 0):03d}"
try:
detail = response.json()
code = detail.get("error", {}).get("code", code)
message = detail.get("error", {}).get("message", response.text)
except ValueError:
message = response.text
raise ShenshiError(code, message)
if response.status_code == 204 or not response.content:
return None
return response.json()
async def get_my_usage(self, user_id: str) -> dict[str, Any]:
data = await self._request_json("GET", "/me/usage", user_id=user_id)
return data or {}
async def list_byok_keys(self, user_id: str) -> list[dict[str, Any]]:
data = await self._request_json("GET", "/me/byok-keys", user_id=user_id)
return (data or {}).get("keys", [])
async def register_byok_key(
self, user_id: str, *, name: str, type: str, base_url: str, api_key: str, model: str
) -> dict[str, Any]:
data = await self._request_json(
"POST",
"/me/byok-keys",
user_id=user_id,
json_body={"name": name, "type": type, "base_url": base_url, "api_key": api_key, "model": model},
)
return data or {}
async def revoke_byok_key(self, user_id: str, key_id: str) -> None:
await self._request_json("DELETE", f"/me/byok-keys/{key_id}", user_id=user_id)
async def stream_chat(self, request: ShenshiChatRequest) -> AsyncIterator[ShenshiDelta]:
payload = request.model_dump(mode="json")
assert payload.get("stream") is True # 数据面当前只有流式形态
+4
View File
@@ -38,6 +38,10 @@ class ShenshiChatRequest(BaseModel):
run_id: str
trace_id: str | None = None
character_id: str | None = None
# 用量归属与配额裁决维度(鉴权后注入;空则记为平台调用)
user_id: str | None = None
# 用户自带密钥引用(配置继承的最终解析结果,神识不自行决定)
byok_key_id: str | None = None
temperature: float | None = None
max_tokens: int | None = None
stream: bool = True
+87
View File
@@ -0,0 +1,87 @@
"""用户面(神识阁数据源)端点测试:BYOK 管理与用量查询,经回声桩客户端端到端。"""
from collections.abc import AsyncGenerator
import pytest
from httpx import ASGITransport, AsyncClient
from app.kernel import Kernel
from app.main import app
from app.services.event_store import DbEventStore
from app.db.session import AsyncSessionLocal
from app.kernel import KernelEventLog
from app.shenshi_client.client import EchoShenshi
@pytest.fixture
async def client(
monkeypatch: pytest.MonkeyPatch, auth_headers: dict[str, str]
) -> AsyncGenerator[AsyncClient, None]:
"""手工装配 Kernel(回声桩客户端)到 app.state(测试不经 lifespan,与既有做法一致)。"""
app.state.kernel = Kernel(
shenshi=EchoShenshi(),
session_factory=AsyncSessionLocal,
event_log=KernelEventLog(DbEventStore(AsyncSessionLocal)),
timeout_seconds=10.0,
)
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test", headers=auth_headers) as c:
yield c
app.state.kernel = None
@pytest.mark.asyncio
async def test_byok_register_list_revoke(client: AsyncClient) -> None:
payload = {
"name": "自带 DeepSeek",
"type": "openai_compatible",
"base_url": "https://api.deepseek.com",
"api_key": "sk-very-secret-value",
"model": "deepseek-v4-flash",
}
created = await client.post("/api/v1/me/byok-keys", json=payload)
assert created.status_code == 201
body = created.json()
# 脱敏:响应不得含明文密钥
assert "sk-very-secret-value" not in body
assert "****" in body.get("api_key_mask", "")
key_id = body["id"]
listed = await client.get("/api/v1/me/byok-keys")
assert listed.status_code == 200
keys = listed.json()["keys"]
assert [k["id"] for k in keys] == [key_id]
assert all("sk-very-secret-value" not in str(k) for k in keys)
revoked = await client.delete(f"/api/v1/me/byok-keys/{key_id}")
assert revoked.status_code == 204
after = await client.get("/api/v1/me/byok-keys")
assert after.json()["keys"] == []
@pytest.mark.asyncio
async def test_byok_register_validation(client: AsyncClient) -> None:
bad = await client.post(
"/api/v1/me/byok-keys",
json={"name": "x", "type": "unknown_type", "base_url": "https://x", "api_key": "k" * 8, "model": "m"},
)
# 类型枚举由神识侧裁决;回声桩直通,这里校验必填字段约束
missing = await client.post("/api/v1/me/byok-keys", json={"name": "x"})
assert missing.status_code == 422
@pytest.mark.asyncio
async def test_my_usage_shape(client: AsyncClient) -> None:
resp = await client.get("/api/v1/me/usage")
assert resp.status_code == 200
body = resp.json()
assert "rows" in body
assert isinstance(body["rows"], list)
@pytest.mark.asyncio
async def test_me_requires_auth() -> None:
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
resp = await c.get("/api/v1/me/byok-keys")
assert resp.status_code in (401, 503) # 未携带凭证被拒(JWT 未配置时 fail-closed)