M4 收口:神识用户面与计量闭环——BYOK/配额三段式/usage 落库/Anthropic+Gemini 适配器/神识阁

- 神识 Go:store 双实现(PG 自动建表+60s 就绪重试 / 内存回退)承载 usage_logs·byok_keys·quota_balances;BYOK 密钥库(AES-GCM 加密、API 脱敏、撤回即失效、主密钥未配置 fail-closed);配额三段式(预扣 max_tokens 钳位 1M→结算实际 token→全败幂等退款,HJ-2004);计量归属(IR user_id→usage_logs,/me/usage 用户维度聚合+配额总览,/admin/usage/sample 对账抽样);Anthropic/Gemini 出站适配器(SSE 解析、system 提升、用量映射);stub 端点补名修复计量空 endpoint
- IR 契约:user_id/byok_key_id 双端同步(Go types.go + Py types.py)
- 灵台:runs.owner_id(迁移 005,server_default 兼容存量)经 kernel 全链透传为 IR user_id;shenshi_client 扩展用户面方法(X-Hunjing-User-Id 代理,Echo 桩带内存实现);/me/byok-keys 登记·列表·撤回与 /me/usage 端点(4 用例)
- 前端神识阁(/models 重建):偏好(生成参数表单)/自带密钥(登记加密保存、脱敏列表、撤回二次确认)/用量(聚合表+配额进度条)三区;me API 库+mock+5 用例;侧栏恢复入口
- 配置与部署:.env.example 增 SHENSHI_DATABASE_URL/SHENSHI_BYOK_MASTER_KEY/SHENSHI_USER_BUDGET_TOKENS;compose 神识容器接自有库(init-shenshi.sql 预建 hunjing_shenshi,与灵台不共库共表)
- 证据:Go 34 用例(BYOK 独占/配额拒绝/结算退款/归属聚合)、后端 532、前端 210、BFF 4 全绿;跨进程 E2E 四场景(灵台 HttpShenshiClient→真 Go 神识:登记脱敏/BYOK 独占不可达整体失败/用量归属聚合/撤回即失效)通过;rewrite-plan M4 标记完成、能力矩阵神识行升已验证
This commit is contained in:
chenxin committed 2026-08-27 18:39:17 +08:00
1 parent 462d570751
commit 144415247e
51 files changed
+3533 -90

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
// PG 实现:与内存实现同语义的直写 SQL。
package store
import (
"database/sql"
"time"
)
type pgStore struct {
db *sql.DB
}
func (p *pgStore) InsertUsage(l UsageLog) error {
_, err := p.db.Exec(
`INSERT INTO usage_logs (id, run_id, character_id, user_id, model, slot, endpoint,
status, error_code, first_token_ms, total_ms, prompt_tokens, completion_tokens, at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`,
l.ID, l.RunID, l.CharacterID, l.UserID, l.Model, l.Slot, l.Endpoint,
l.Status, l.ErrorCode, l.FirstTokenMS, l.TotalMS, l.PromptTokens, l.CompletionTokens, l.At,
)
return err
}
func (p *pgStore) AggregateUsage(userID string) ([]UsageAgg, error) {
q := `SELECT user_id, model, endpoint, COUNT(*),
COUNT(*) FILTER (WHERE status = 'ok'),
COALESCE(SUM(prompt_tokens) FILTER (WHERE status = 'ok'), 0),
COALESCE(SUM(completion_tokens) FILTER (WHERE status = 'ok'), 0)
FROM usage_logs`
args := []any{}
if userID != "" {
q += ` WHERE user_id = $1`
args = append(args, userID)
}
q += ` GROUP BY user_id, model, endpoint ORDER BY user_id, model`
rows, err := p.db.Query(q, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []UsageAgg
for rows.Next() {
var r UsageAgg
if err := rows.Scan(&r.UserID, &r.Model, &r.Endpoint, &r.Requests, &r.OK, &r.PromptTokens, &r.CompletionTokens); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
func (p *pgStore) SampleUsage(limit int) ([]UsageLog, error) {
rows, err := p.db.Query(
`SELECT id, run_id, character_id, user_id, model, slot, endpoint, status, error_code,
first_token_ms, total_ms, prompt_tokens, completion_tokens, at
FROM usage_logs ORDER BY at DESC LIMIT $1`, limit)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []UsageLog
for rows.Next() {
var l UsageLog
if err := rows.Scan(&l.ID, &l.RunID, &l.CharacterID, &l.UserID, &l.Model, &l.Slot, &l.Endpoint,
&l.Status, &l.ErrorCode, &l.FirstTokenMS, &l.TotalMS, &l.PromptTokens, &l.CompletionTokens, &l.At); err != nil {
return nil, err
}
out = append(out, l)
}
return out, rows.Err()
}
func (p *pgStore) UpsertByok(k ByokKey) error {
_, err := p.db.Exec(
`INSERT INTO byok_keys (id, user_id, name, type, base_url, api_key, model, created_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
ON CONFLICT (id) DO UPDATE SET name=$3, type=$4, base_url=$5, api_key=$6, model=$7`,
k.ID, k.UserID, k.Name, k.Type, k.BaseURL, k.APIKey, k.Model, k.CreatedAt)
return err
}
func (p *pgStore) ListByok(userID string) ([]ByokKey, error) {
return p.byokQuery(`SELECT id, user_id, name, type, base_url, api_key, model, created_at, last_used, revoked_at
FROM byok_keys WHERE user_id = $1 AND revoked_at IS NULL ORDER BY created_at`, userID)
}
func (p *pgStore) GetByok(userID, id string) (*ByokKey, error) {
keys, err := p.byokQuery(`SELECT id, user_id, name, type, base_url, api_key, model, created_at, last_used, revoked_at
FROM byok_keys WHERE user_id = $1 AND id = $2 AND revoked_at IS NULL`, userID, id)
if err != nil || len(keys) == 0 {
return nil, err
}
return &keys[0], nil
}
func (p *pgStore) byokQuery(q string, args ...any) ([]ByokKey, error) {
rows, err := p.db.Query(q, args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []ByokKey
for rows.Next() {
var k ByokKey
if err := rows.Scan(&k.ID, &k.UserID, &k.Name, &k.Type, &k.BaseURL, &k.APIKey, &k.Model, &k.CreatedAt, &k.LastUsed, &k.RevokedAt); err != nil {
return nil, err
}
out = append(out, k)
}
return out, rows.Err()
}
func (p *pgStore) RevokeByok(userID, id string) error {
res, err := p.db.Exec(`UPDATE byok_keys SET revoked_at = $1 WHERE user_id = $2 AND id = $3 AND revoked_at IS NULL`,
time.Now().UTC(), userID, id)
if err != nil {
return err
}
if n, _ := res.RowsAffected(); n == 0 {
return ErrByokNotFound
}
return nil
}
func (p *pgStore) TouchByok(id string, at time.Time) error {
_, err := p.db.Exec(`UPDATE byok_keys SET last_used = $1 WHERE id = $2`, at, id)
return err
}
func (p *pgStore) Reserve(userID string, amount int64) error {
if amount <= 0 {
return nil
}
// UPSERT + 原子累加;配额上限裁决在 quota 包(集中单一函数)
_, err := p.db.Exec(`INSERT INTO quota_balances (user_id, reserved, used) VALUES ($1, $2, 0)
ON CONFLICT (user_id) DO UPDATE SET reserved = quota_balances.reserved + $2`, userID, amount)
return err
}
func (p *pgStore) Settle(userID string, reserved, actual int64) error {
// 饱和截断在 SQL 层以 GREATEST 保证(防负)
_, err := p.db.Exec(`UPDATE quota_balances SET
reserved = GREATEST(reserved - $2, 0), used = used + $3 WHERE user_id = $1`,
userID, reserved, actual)
return err
}
func (p *pgStore) Refund(userID string, reserved int64) error {
_, err := p.db.Exec(`UPDATE quota_balances SET reserved = GREATEST(reserved - $2, 0) WHERE user_id = $1`,
userID, reserved)
return err
}
func (p *pgStore) Balance(userID string) (QuotaBalance, error) {
var b QuotaBalance
b.UserID = userID
err := p.db.QueryRow(`SELECT reserved, used FROM quota_balances WHERE user_id = $1`, userID).
Scan(&b.Reserved, &b.Used)
if err == sql.ErrNoRows {
return b, nil
}
return b, err
}
+343
View File
@@ -0,0 +1,343 @@
// Package store 神识自有库访问层:usage_logs / byok_keys / quota 余额。
//
// 双实现:PG(SHENSHI_DATABASE_URL,生产)与内存(开发/测试,未配置时回退)。
// 语义契约由 store_test.go 在内存实现上钉死;PG 实现为同接口直写 SQL。
package store
import (
"database/sql"
"errors"
"fmt"
"sync"
"time"
// PG 驱动(database/sql 标准接口)
_ "github.com/jackc/pgx/v5/stdlib"
)
// UsageLog 一次端点尝试的完整计量记录(shenshi-gateway.md §3 usage_logs)。
type UsageLog struct {
ID string // 尝试唯一标识(run_id#endpoint#seq,配额结算幂等键)
RunID string
CharacterID string
UserID string
Model string
Slot string
Endpoint string
Status string // ok / error / first_byte_timeout
ErrorCode string
FirstTokenMS int64
TotalMS int64
PromptTokens int
CompletionTokens int
At time.Time
}
// UsageAgg 用量聚合行(/me/usage)。
type UsageAgg struct {
UserID string `json:"user_id"`
Model string `json:"model"`
Endpoint string `json:"endpoint"`
Requests int `json:"requests"`
OK int `json:"ok"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
}
// ByokKey 用户自带密钥(API 永远脱敏返回)。
type ByokKey struct {
ID string
UserID string
Name string
Type string // openai_compatible / anthropic / gemini(契约词典 §14.2)
BaseURL string
APIKey string // 加密存储;出站时解密
Model string
CreatedAt time.Time
LastUsed *time.Time
RevokedAt *time.Time
}
// QuotaBalance 用户配额账(三段式状态:Reserved 预扣 / Used 已结算)。
type QuotaBalance struct {
UserID string
Reserved int64
Used int64
}
// ErrByokNotFound BYOK 密钥不存在或已撤回。
var ErrByokNotFound = errors.New("BYOK 密钥不存在或已撤回")
// Store 存储接口(全异步语义由调用方保证;实现自身须并发安全)。
type Store interface {
InsertUsage(log UsageLog) error
AggregateUsage(userID string) ([]UsageAgg, error)
SampleUsage(limit int) ([]UsageLog, error) // 对账抽样:最近 N 条原始记录
UpsertByok(key ByokKey) error
ListByok(userID string) ([]ByokKey, error)
GetByok(userID, id string) (*ByokKey, error)
RevokeByok(userID, id string) error
TouchByok(id string, at time.Time) error
// Quota 三段式原子操作:Reserve 预扣(不足返回错误);Settle 结算差值;Refund 幂等退款。
Reserve(userID string, amount int64) error
Settle(userID string, reserved, actual int64) error
Refund(userID string, reserved int64) error
Balance(userID string) (QuotaBalance, error)
}
// Open 按配置打开存储:DSN 非空时 PG(自动建表,含就绪重试),否则内存实现。
// 重试窗口 60 秒:容器编排下 PG 可能晚于神识就绪(跨 compose 文件无 depends_on)。
func Open(dsn string) (Store, error) {
if dsn == "" {
return NewMem(), nil
}
db, err := sql.Open("pgx", dsn)
if err != nil {
return nil, fmt.Errorf("打开神识库失败: %w", err)
}
deadline := time.Now().Add(60 * time.Second)
for {
err = migrate(db)
if err == nil {
return &pgStore{db: db}, nil
}
if time.Now().After(deadline) {
return nil, fmt.Errorf("神识库迁移失败(60 秒重试后放弃): %w", err)
}
time.Sleep(2 * time.Second)
}
}
func migrate(db *sql.DB) error {
stmts := []string{
`CREATE TABLE IF NOT EXISTS usage_logs (
id TEXT PRIMARY KEY,
run_id TEXT NOT NULL,
character_id TEXT NOT NULL DEFAULT '',
user_id TEXT NOT NULL DEFAULT '',
model TEXT NOT NULL,
slot TEXT NOT NULL,
endpoint TEXT NOT NULL,
status TEXT NOT NULL,
error_code TEXT NOT NULL DEFAULT '',
first_token_ms BIGINT NOT NULL DEFAULT 0,
total_ms BIGINT NOT NULL DEFAULT 0,
prompt_tokens INT NOT NULL DEFAULT 0,
completion_tokens INT NOT NULL DEFAULT 0,
at TIMESTAMPTZ NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_usage_logs_user ON usage_logs(user_id, at DESC)`,
`CREATE TABLE IF NOT EXISTS byok_keys (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL,
base_url TEXT NOT NULL DEFAULT '',
api_key TEXT NOT NULL,
model TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL,
last_used TIMESTAMPTZ,
revoked_at TIMESTAMPTZ
)`,
`CREATE INDEX IF NOT EXISTS idx_byok_user ON byok_keys(user_id)`,
`CREATE TABLE IF NOT EXISTS quota_balances (
user_id TEXT PRIMARY KEY,
reserved BIGINT NOT NULL DEFAULT 0,
used BIGINT NOT NULL DEFAULT 0
)`,
}
for _, s := range stmts {
if _, err := db.Exec(s); err != nil {
return fmt.Errorf("建表失败: %w", err)
}
}
return nil
}
// ── 内存实现 ──────────────────────────────────────────────
type memStore struct {
mu sync.Mutex
usage []UsageLog
byok map[string][]ByokKey // user → keys
quota map[string]*QuotaBalance
}
// NewMem 内存存储(保留最近 10 万条 usage,防无界增长)。
func NewMem() Store {
return &memStore{byok: map[string][]ByokKey{}, quota: map[string]*QuotaBalance{}}
}
const memUsageLimit = 100_000
func (m *memStore) InsertUsage(log UsageLog) error {
m.mu.Lock()
defer m.mu.Unlock()
if len(m.usage) >= memUsageLimit {
m.usage = m.usage[1:]
}
m.usage = append(m.usage, log)
return nil
}
func (m *memStore) AggregateUsage(userID string) ([]UsageAgg, error) {
m.mu.Lock()
defer m.mu.Unlock()
agg := map[string]*UsageAgg{}
for _, e := range m.usage {
if userID != "" && e.UserID != userID {
continue
}
key := e.UserID + "|" + e.Model + "|" + e.Endpoint
row := agg[key]
if row == nil {
row = &UsageAgg{UserID: e.UserID, Model: e.Model, Endpoint: e.Endpoint}
agg[key] = row
}
row.Requests++
if e.Status == "ok" {
row.OK++
row.PromptTokens += e.PromptTokens
row.CompletionTokens += e.CompletionTokens
}
}
out := make([]UsageAgg, 0, len(agg))
for _, row := range agg {
out = append(out, *row)
}
return out, nil
}
func (m *memStore) SampleUsage(limit int) ([]UsageLog, error) {
m.mu.Lock()
defer m.mu.Unlock()
n := limit
if n > len(m.usage) {
n = len(m.usage)
}
out := make([]UsageLog, n)
copy(out, m.usage[len(m.usage)-n:])
return out, nil
}
func (m *memStore) UpsertByok(key ByokKey) error {
m.mu.Lock()
defer m.mu.Unlock()
keys := m.byok[key.UserID]
for i, k := range keys {
if k.ID == key.ID {
keys[i] = key
return nil
}
}
m.byok[key.UserID] = append(keys, key)
return nil
}
func (m *memStore) ListByok(userID string) ([]ByokKey, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := make([]ByokKey, 0, len(m.byok[userID]))
for _, k := range m.byok[userID] {
if k.RevokedAt == nil {
out = append(out, k)
}
}
return out, nil
}
func (m *memStore) GetByok(userID, id string) (*ByokKey, error) {
m.mu.Lock()
defer m.mu.Unlock()
for _, k := range m.byok[userID] {
if k.ID == id && k.RevokedAt == nil {
out := k
return &out, nil
}
}
return nil, nil
}
func (m *memStore) RevokeByok(userID, id string) error {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now().UTC()
keys := m.byok[userID]
for i, k := range keys {
if k.ID == id && k.RevokedAt == nil {
keys[i].RevokedAt = &now
return nil
}
}
return ErrByokNotFound
}
func (m *memStore) TouchByok(id string, at time.Time) error {
m.mu.Lock()
defer m.mu.Unlock()
for user, keys := range m.byok {
for i, k := range keys {
if k.ID == id {
used := at
keys[i].LastUsed = &used
m.byok[user] = keys
return nil
}
}
}
return nil
}
func (m *memStore) Reserve(userID string, amount int64) error {
if amount <= 0 {
return nil
}
m.mu.Lock()
defer m.mu.Unlock()
b := m.quotaGet(userID)
b.Reserved += amount
return nil
}
func (m *memStore) Settle(userID string, reserved, actual int64) error {
m.mu.Lock()
defer m.mu.Unlock()
b := m.quotaGet(userID)
b.Reserved -= reserved
if b.Reserved < 0 {
b.Reserved = 0
}
b.Used += actual
return nil
}
func (m *memStore) Refund(userID string, reserved int64) error {
m.mu.Lock()
defer m.mu.Unlock()
b := m.quotaGet(userID)
b.Reserved -= reserved
if b.Reserved < 0 {
b.Reserved = 0
}
return nil
}
func (m *memStore) Balance(userID string) (QuotaBalance, error) {
m.mu.Lock()
defer m.mu.Unlock()
if b, ok := m.quota[userID]; ok {
return *b, nil
}
return QuotaBalance{UserID: userID}, nil
}
func (m *memStore) quotaGet(userID string) *QuotaBalance {
b, ok := m.quota[userID]
if !ok {
b = &QuotaBalance{UserID: userID}
m.quota[userID] = b
}
return b
}
+105
View File
@@ -0,0 +1,105 @@
// store 语义契约测试(内存实现;PG 实现同接口同语义)。
package store
import (
"testing"
"time"
)
func newUsage(id, user, model, endpoint, status string, p, c int) UsageLog {
return UsageLog{ID: id, RunID: "run_t", UserID: user, Model: model, Slot: "chat",
Endpoint: endpoint, Status: status, PromptTokens: p, CompletionTokens: c,
At: time.Now().UTC()}
}
func TestUsageAggregateFiltersByUserAndCountsOK(t *testing.T) {
st := NewMem()
_ = st.InsertUsage(newUsage("1", "u1", "m", "e1", "ok", 10, 20))
_ = st.InsertUsage(newUsage("2", "u1", "m", "e1", "error", 0, 0))
_ = st.InsertUsage(newUsage("3", "u2", "m", "e2", "ok", 1, 2))
rows, err := st.AggregateUsage("u1")
if err != nil {
t.Fatal(err)
}
if len(rows) != 1 {
t.Fatalf("u1 聚合行数 = %d, 期望 1", len(rows))
}
r := rows[0]
if r.Requests != 2 || r.OK != 1 || r.PromptTokens != 10 || r.CompletionTokens != 20 {
t.Fatalf("聚合字段错误: %+v", r)
}
all, _ := st.AggregateUsage("")
if len(all) != 2 {
t.Fatalf("全体聚合行数 = %d, 期望 2", len(all))
}
}
func TestUsageSampleReturnsLatest(t *testing.T) {
st := NewMem()
for i := 0; i < 5; i++ {
_ = st.InsertUsage(newUsage(string(rune('a'+i)), "u1", "m", "e", "ok", i, 0))
}
rows, _ := st.SampleUsage(2)
if len(rows) != 2 || rows[1].ID != "e" {
t.Fatalf("抽样应返回最新 2 条(d,e),实际 %+v", rows)
}
}
func TestByokLifecycle(t *testing.T) {
st := NewMem()
key := ByokKey{ID: "byok_x", UserID: "u1", Name: "主力", Type: "openai_compatible",
BaseURL: "https://api.example.com", APIKey: "enc", Model: "m1", CreatedAt: time.Now().UTC()}
if err := st.UpsertByok(key); err != nil {
t.Fatal(err)
}
got, err := st.GetByok("u1", "byok_x")
if err != nil || got == nil || got.Model != "m1" {
t.Fatalf("GetByok 失败: %v %+v", err, got)
}
// 跨用户隔离
if other, _ := st.GetByok("u2", "byok_x"); other != nil {
t.Fatal("u2 不应读到 u1 的密钥")
}
if err := st.RevokeByok("u1", "byok_x"); err != nil {
t.Fatal(err)
}
got2, _ := st.GetByok("u1", "byok_x")
if got2 != nil {
t.Fatal("撤回后不应可读")
}
if err := st.RevokeByok("u1", "byok_x"); err != ErrByokNotFound {
t.Fatalf("重复撤回应返回 ErrByokNotFound,实际 %v", err)
}
}
func TestQuotaSaturatingRefund(t *testing.T) {
st := NewMem()
if err := st.Reserve("u1", 100); err != nil {
t.Fatal(err)
}
// 幂等退款:重复退款不产生负余额
if err := st.Refund("u1", 100); err != nil {
t.Fatal(err)
}
if err := st.Refund("u1", 100); err != nil {
t.Fatal(err)
}
b, _ := st.Balance("u1")
if b.Reserved < 0 || b.Used != 0 {
t.Fatalf("重复退款后余额为负: %+v", b)
}
}
func TestQuotaSettleReleasesDifference(t *testing.T) {
st := NewMem()
_ = st.Reserve("u1", 100)
if err := st.Settle("u1", 100, 37); err != nil {
t.Fatal(err)
}
b, _ := st.Balance("u1")
if b.Reserved != 0 || b.Used != 37 {
t.Fatalf("结算后应释放全部预扣并记实际用量: %+v", b)
}
}