✨ feat(database): 完善请求日志体系并为全表补充时间字段

- request_log 新增 user_agent 列(含旧表 ALTER 迁移)及 ip/status 索引,记录解码后的明文 query(含 key,不脱敏)
- 请求日志批量写入改为多值 INSERT 单次往返,替代事务逐条 Exec
- 新增按本地时区分天聚合查询,状态页展示今日请求量与近 7 天请求趋势(峰值日高亮)
- 新增 TZ 时区支持:.env 配置 TZ 后日志时间戳与状态页全部时间统一生效,二进制内嵌 tzdata 保证容器可用
- 五张表统一补齐 created_at/updated_at 字段,启动时自动补列并按业务时间回填旧行
- device_pool 的 device_header 增加唯一约束;audio_cache 写入改为 UPSERT 修复同 key 重复报错
- 状态页"最近请求"新增 UA 列与限流状态显示;config.yaml 暴露 rate_limit_qps 配置并默认启用请求日志
- deploy.sh 同名版本默认拒绝覆盖,新增 --force/-f 参数显式允许覆盖
This commit is contained in:
chenxin committed 2026-10-09 22:12:17 +08:00
1 parent 1a4997e031
commit e3d96ca637
13 files changed
+288 -55

No files matched your search

+3
View File
@@ -1,6 +1,9 @@
# 敏感配置模板:复制为 .env 并填入真实值(.env 已被 gitignore,不会提交) # 敏感配置模板:复制为 .env 并填入真实值(.env 已被 gitignore,不会提交)
# 环境变量优先于 config.yaml 中的对应配置 # 环境变量优先于 config.yaml 中的对应配置
# 进程时区(IANA 名称),日志时间戳与 /status 页所有时间均按此显示
TZ=Asia/Shanghai
# 服务访问密钥(/status、/self-info 等端点的鉴权) # 服务访问密钥(/status、/self-info 等端点的鉴权)
# FQ_SERVER_ACCESS_KEY=your-access-key # FQ_SERVER_ACCESS_KEY=your-access-key
+6
View File
@@ -14,6 +14,9 @@ import (
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
// 嵌入 tzdata,容器内无系统时区库时 TZ 仍可用
_ "time/tzdata"
) )
// Version / BuildTime 由编译时 -ldflags "-X main.Version=... -X main.BuildTime=..." 注入 // Version / BuildTime 由编译时 -ldflags "-X main.Version=... -X main.BuildTime=..." 注入
@@ -26,6 +29,9 @@ var (
) )
func main() { func main() {
// 时区生效须先于一切时间显示(.env 中的 TZ)
config.ApplyTimezone()
logger.Info(fmt.Sprintf("番茄小说API服务 版本=%s 编译时间=%s", Version, BuildTime)) logger.Info(fmt.Sprintf("番茄小说API服务 版本=%s 编译时间=%s", Version, BuildTime))
// 加载配置文件 // 加载配置文件
+3 -1
View File
@@ -5,10 +5,12 @@ server:
port: "40180" # 监听端口 port: "40180" # 监听端口
host: "0.0.0.0" # 监听地址 host: "0.0.0.0" # 监听地址
access_key: "" # 访问密钥,为空则不启用验证(用于 /status) access_key: "" # 访问密钥,为空则不启用验证(用于 /status)
rate_limit_qps: 0 # 每 IP 每秒请求上限(令牌桶,突发 10),0=不限制
# 请求日志(只记元数据与错误摘要,不存响应体;在 /status 页"最近请求"展示) # 请求日志(只记元数据与错误摘要,不存响应体;在 /status 页"最近请求"展示)
# 注意:日志时间显示与"近 7 天请求"统计依赖 .env 中的 TZ 时区
request_log: request_log:
enabled: false enabled: true
retention_days: 7 # 日志保留天数,自动清理 retention_days: 7 # 日志保留天数,自动清理
# 番茄小说配置 # 番茄小说配置
+11
View File
@@ -60,6 +60,7 @@ BUILD_ONLY=false
ROLLBACK=false ROLLBACK=false
ROLLBACK_VERSION="" ROLLBACK_VERSION=""
LIST=false LIST=false
FORCE=false
print_help() { print_help() {
cat <<EOF cat <<EOF
@@ -69,6 +70,7 @@ print_help() {
--init 首次初始化,写入 systemd 和 nginx 配置 --init 首次初始化,写入 systemd 和 nginx 配置
--no-pull 跳过 git pull --no-pull 跳过 git pull
--build-only 只编译,不部署、不重启服务 --build-only 只编译,不部署、不重启服务
--force, -f 允许覆盖已存在的同名版本(默认报错退出)
--rollback [版本] 回滚到指定版本(不带版本号则交互式选择) --rollback [版本] 回滚到指定版本(不带版本号则交互式选择)
--list 列出所有已部署的版本 --list 列出所有已部署的版本
--help, -h 显示此帮助 --help, -h 显示此帮助
@@ -101,6 +103,7 @@ while [ $i -lt ${#ARGS[@]} ]; do
fi fi
;; ;;
--list) LIST=true ;; --list) LIST=true ;;
--force|-f) FORCE=true ;;
--help|-h) print_help; exit 0 ;; --help|-h) print_help; exit 0 ;;
v*) VERSION="$arg" ;; v*) VERSION="$arg" ;;
*) err "未知参数: $arg"; print_help; exit 1 ;; *) err "未知参数: $arg"; print_help; exit 1 ;;
@@ -280,6 +283,14 @@ fi
# ==================== 4. 部署二进制 ==================== # ==================== 4. 部署二进制 ====================
log "部署到 ${BIN_DIR}..." log "部署到 ${BIN_DIR}..."
$SUDO mkdir -p "${BIN_DIR}" $SUDO mkdir -p "${BIN_DIR}"
# 同名版本默认拒绝覆盖,防止 --rollback 回滚到被覆盖的旧文件
if [ -e "${BIN_DIR}/${APP_NAME}-${VERSION}" ]; then
if [ "$FORCE" = true ]; then
warn "版本 ${VERSION} 已存在,--force 覆盖"
else
err "版本 ${VERSION} 已存在于 ${BIN_DIR},换版本号或加 --force 覆盖"; exit 1
fi
fi
$SUDO mv "${APP_NAME}-${VERSION}" "${BIN_DIR}/" $SUDO mv "${APP_NAME}-${VERSION}" "${BIN_DIR}/"
$SUDO chmod +x "${BIN_DIR}/${APP_NAME}-${VERSION}" $SUDO chmod +x "${BIN_DIR}/${APP_NAME}-${VERSION}"
+18
View File
@@ -3,8 +3,10 @@ package config
import ( import (
"fmt" "fmt"
"log" "log"
"os"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
@@ -121,6 +123,22 @@ func NewConfigManager() *ConfigManager {
} }
} }
// ApplyTimezone 按 TZ 环境变量/.env 设置进程时区。
// 必须在任何时间显示之前调用(time.Local 首次使用后不再重读 TZ)。
// 环境变量已设 TZ 时以环境变量为准(.env 不覆盖已有环境变量)。
func ApplyTimezone() {
loadDotenv("./.env")
tz := os.Getenv("TZ")
if tz == "" {
return
}
if _, err := time.LoadLocation(tz); err != nil {
log.Printf("[config/WARN] 无效时区 TZ=%s,沿用系统时区: %v", tz, err)
return // 不写入 TZ:无效值会让 time.Local 落到 UTC
}
os.Setenv("TZ", tz)
}
// LoadConfig 加载配置文件 // LoadConfig 加载配置文件
func LoadConfig(configPath string) error { func LoadConfig(configPath string) error {
// 初始化管理器 // 初始化管理器
+5 -3
View File
@@ -38,9 +38,11 @@ func GenerateAudioProxyURL(originalURL, deviceID string, headers map[string]stri
if cacheDB != nil { if cacheDB != nil {
_, err := cacheDB.Exec(q(` _, err := cacheDB.Exec(q(`
INSERT INTO audio_cache (key, original_url, device_id, headers, created_at) INSERT INTO audio_cache (key, original_url, device_id, headers, created_at, updated_at)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
`), key, originalURL, deviceID, string(headersJSON), time.Now().Unix()) ON CONFLICT(key) DO UPDATE SET original_url = excluded.original_url,
device_id = excluded.device_id, headers = excluded.headers, updated_at = excluded.updated_at
`), key, originalURL, deviceID, string(headersJSON), time.Now().Unix(), time.Now().Unix())
if err != nil { if err != nil {
audioLogger.Error("保存音频缓存失败: " + err.Error()) audioLogger.Error("保存音频缓存失败: " + err.Error())
} }
+56 -11
View File
@@ -149,12 +149,14 @@ func createTables() error {
`CREATE TABLE IF NOT EXISTS cache ( `CREATE TABLE IF NOT EXISTS cache (
key TEXT PRIMARY KEY, key TEXT PRIMARY KEY,
value TEXT, value TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now')) created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`, )`,
`CREATE TABLE IF NOT EXISTS cache ( `CREATE TABLE IF NOT EXISTS cache (
key TEXT PRIMARY KEY, key TEXT PRIMARY KEY,
value TEXT, value TEXT,
created_at INTEGER DEFAULT 0 created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
)`}, )`},
{"statistics", {"statistics",
`CREATE TABLE IF NOT EXISTS statistics ( `CREATE TABLE IF NOT EXISTS statistics (
@@ -168,6 +170,7 @@ func createTables() error {
audio_requests INTEGER DEFAULT 0, audio_requests INTEGER DEFAULT 0,
video_requests INTEGER DEFAULT 0, video_requests INTEGER DEFAULT 0,
category_requests INTEGER DEFAULT 0, category_requests INTEGER DEFAULT 0,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now')) updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`, )`,
`CREATE TABLE IF NOT EXISTS statistics ( `CREATE TABLE IF NOT EXISTS statistics (
@@ -181,26 +184,29 @@ func createTables() error {
audio_requests INTEGER DEFAULT 0, audio_requests INTEGER DEFAULT 0,
video_requests INTEGER DEFAULT 0, video_requests INTEGER DEFAULT 0,
category_requests INTEGER DEFAULT 0, category_requests INTEGER DEFAULT 0,
updated_at INTEGER DEFAULT 0 created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
)`}, )`},
{"device_pool", {"device_pool",
`CREATE TABLE IF NOT EXISTS device_pool ( `CREATE TABLE IF NOT EXISTS device_pool (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
device_header TEXT NOT NULL, device_header TEXT NOT NULL UNIQUE,
key TEXT NOT NULL, key TEXT NOT NULL,
key_ver INTEGER NOT NULL, key_ver INTEGER NOT NULL,
is_valid INTEGER DEFAULT 1, is_valid INTEGER DEFAULT 1,
created_at INTEGER NOT NULL, created_at INTEGER NOT NULL,
last_used_at INTEGER NOT NULL last_used_at INTEGER NOT NULL,
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`, )`,
`CREATE TABLE IF NOT EXISTS device_pool ( `CREATE TABLE IF NOT EXISTS device_pool (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
device_header TEXT NOT NULL, device_header TEXT NOT NULL UNIQUE,
key TEXT NOT NULL, key TEXT NOT NULL,
key_ver INTEGER NOT NULL, key_ver INTEGER NOT NULL,
is_valid INTEGER DEFAULT 1, is_valid INTEGER DEFAULT 1,
created_at INTEGER NOT NULL, created_at INTEGER NOT NULL,
last_used_at INTEGER NOT NULL last_used_at INTEGER NOT NULL,
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
)`}, )`},
{"audio_cache", {"audio_cache",
`CREATE TABLE IF NOT EXISTS audio_cache ( `CREATE TABLE IF NOT EXISTS audio_cache (
@@ -208,14 +214,16 @@ func createTables() error {
original_url TEXT NOT NULL, original_url TEXT NOT NULL,
device_id TEXT NOT NULL, device_id TEXT NOT NULL,
headers TEXT, headers TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now')) created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`, )`,
`CREATE TABLE IF NOT EXISTS audio_cache ( `CREATE TABLE IF NOT EXISTS audio_cache (
key TEXT PRIMARY KEY, key TEXT PRIMARY KEY,
original_url TEXT NOT NULL, original_url TEXT NOT NULL,
device_id TEXT NOT NULL, device_id TEXT NOT NULL,
headers TEXT, headers TEXT,
created_at INTEGER DEFAULT 0 created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
)`}, )`},
} }
@@ -242,6 +250,42 @@ func createTables() error {
if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`); err != nil { if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`); err != nil {
cacheLogger.Warning("创建device_pool索引失败: " + err.Error()) cacheLogger.Warning("创建device_pool索引失败: " + err.Error())
} }
// 旧库补唯一约束:表已存在时 DDL 的 UNIQUE 不会生效,用唯一索引补齐;
// 若存量数据已有重复 device_header 则创建失败,仅告警提示人工清理
if _, err := cacheDB.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_device_pool_header ON device_pool(device_header)`); err != nil {
cacheLogger.Warning("补建device_header唯一索引失败(可能存在重复设备,清理后重启即可): " + err.Error())
}
// 旧表补 created_at/updated_at 列(新表 DDL 自带,此处仅老库生效)
for _, m := range []struct{ table, col, ddl string }{
{"cache", "updated_at", "INTEGER DEFAULT 0"}, // SQLite ALTER 不允许非常量默认值,先补 0 再回填
{"statistics", "created_at", "INTEGER DEFAULT 0"},
{"device_pool", "updated_at", "INTEGER DEFAULT 0"},
{"audio_cache", "updated_at", "INTEGER DEFAULT 0"},
} {
if dbDriver == "pgx" {
m.ddl = "BIGINT DEFAULT 0"
}
alter := "ALTER TABLE " + m.table + " ADD COLUMN "
if dbDriver == "pgx" {
alter += "IF NOT EXISTS "
}
if _, err := cacheDB.Exec(alter + m.col + " " + m.ddl); err != nil {
cacheLogger.Debug("补列跳过(已存在): " + m.table + "." + m.col)
}
}
// 旧行回填(幂等:仅 0 值行受影响)
for _, backfill := range []string{
`UPDATE cache SET updated_at = created_at WHERE updated_at = 0`,
`UPDATE statistics SET created_at = updated_at WHERE created_at = 0`,
`UPDATE device_pool SET updated_at = last_used_at WHERE updated_at = 0`,
`UPDATE audio_cache SET updated_at = created_at WHERE updated_at = 0`,
} {
if _, err := cacheDB.Exec(backfill); err != nil {
cacheLogger.Warning("时间列回填失败: " + err.Error())
}
}
return nil return nil
} }
@@ -337,8 +381,9 @@ func UpdateCache(key string, value string, expi int) error {
} }
// UPSERT:ON CONFLICT 语法 SQLite 3.24+ 与 PostgreSQL 通用 // UPSERT:ON CONFLICT 语法 SQLite 3.24+ 与 PostgreSQL 通用
_, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at) VALUES (?, ?, ?) _, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at, updated_at) VALUES (?, ?, ?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value`), key, string(jsonData), time.Now().Unix()) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`),
key, string(jsonData), time.Now().Unix(), time.Now().Unix())
if err != nil { if err != nil {
cacheLogger.Error("更新缓存失败: " + err.Error()) cacheLogger.Error("更新缓存失败: " + err.Error())
return err return err
+4 -2
View File
@@ -30,7 +30,8 @@ func TestConcurrentCacheAccess(t *testing.T) {
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache ( if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache (
key TEXT PRIMARY KEY, key TEXT PRIMARY KEY,
value TEXT, value TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now')) created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`); err != nil { )`); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -73,7 +74,8 @@ func TestWithCache(t *testing.T) {
}() }()
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache ( if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache (
key TEXT PRIMARY KEY, value TEXT, key TEXT PRIMARY KEY, value TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now')) created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)`); err != nil { )`); err != nil {
t.Fatal(err) t.Fatal(err)
} }
+5 -5
View File
@@ -401,11 +401,11 @@ func (p *DeviceKeyPool) saveDeviceToDB(item *DeviceKeyItem) error {
// RETURNING id 同时兼容 SQLite(3.35+) 与 PostgreSQL,替代 LastInsertId() // RETURNING id 同时兼容 SQLite(3.35+) 与 PostgreSQL,替代 LastInsertId()
err := cacheDB.QueryRow(q(` err := cacheDB.QueryRow(q(`
INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at) INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING id RETURNING id
`), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid),
item.CreatedAt.Unix(), item.LastUsedAt.Unix()).Scan(&item.ID) item.CreatedAt.Unix(), item.LastUsedAt.Unix(), item.CreatedAt.Unix()).Scan(&item.ID)
return err return err
} }
@@ -418,10 +418,10 @@ func (p *DeviceKeyPool) updateDeviceInDB(item *DeviceKeyItem) error {
_, err := cacheDB.Exec(q(` _, err := cacheDB.Exec(q(`
UPDATE device_pool UPDATE device_pool
SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ? SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ?, updated_at = ?
WHERE id = ? WHERE id = ?
`), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid),
item.LastUsedAt.Unix(), item.ID) item.LastUsedAt.Unix(), item.LastUsedAt.Unix(), item.ID)
return err return err
} }
+120 -23
View File
@@ -3,6 +3,7 @@ package fqapi
import ( import (
"fanqie/internal/config" "fanqie/internal/config"
"fmt" "fmt"
"strings"
"time" "time"
) )
@@ -15,10 +16,11 @@ type RequestLogEntry struct {
IP string `json:"ip"` IP string `json:"ip"`
Method string `json:"method"` Method string `json:"method"`
Path string `json:"path"` Path string `json:"path"`
Query string `json:"query"` // 已脱敏(key 参数替换为 ***) Query string `json:"query"` // 解码后查询串(含 key,不脱敏)
Status int `json:"status"` Status int `json:"status"`
DurationMs int64 `json:"duration_ms"` DurationMs int64 `json:"duration_ms"`
ErrorMsg string `json:"error_msg"` // 仅 status>=400 时截断保留 ErrorMsg string `json:"error_msg"` // 仅 status>=400 时截断保留
UserAgent string `json:"user_agent"` // 客户端 UA,排障时区分调用方
} }
// ErrMsgMax 错误响应摘要的最大保留字节数 // ErrMsgMax 错误响应摘要的最大保留字节数
@@ -52,7 +54,10 @@ func InitRequestLog() error {
query TEXT, query TEXT,
status INTEGER, status INTEGER,
duration_ms INTEGER, duration_ms INTEGER,
error_msg TEXT error_msg TEXT,
user_agent TEXT,
created_at INTEGER DEFAULT (strftime('%s', 'now')),
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
)` )`
if dbDriver == "pgx" { if dbDriver == "pgx" {
ddl = `CREATE TABLE IF NOT EXISTS request_log ( ddl = `CREATE TABLE IF NOT EXISTS request_log (
@@ -64,14 +69,49 @@ func InitRequestLog() error {
query TEXT, query TEXT,
status INTEGER, status INTEGER,
duration_ms BIGINT, duration_ms BIGINT,
error_msg TEXT error_msg TEXT,
user_agent TEXT,
created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
)` )`
} }
if _, err := cacheDB.Exec(ddl); err != nil { if _, err := cacheDB.Exec(ddl); err != nil {
return fmt.Errorf("创建request_log表失败: %w", err) return fmt.Errorf("创建request_log表失败: %w", err)
} }
if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_request_log_ts ON request_log(ts)`); err != nil { // 旧表补列(表已存在时 CREATE 不会加列;PG 支持 IF NOT EXISTS,SQLite 报错即视为已存在)
cacheLogger.Warning("创建request_log索引失败: " + err.Error()) if dbDriver == "pgx" {
for _, col := range []string{
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS user_agent TEXT`,
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS created_at BIGINT DEFAULT 0`,
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS updated_at BIGINT DEFAULT 0`,
} {
if _, err := cacheDB.Exec(col); err != nil {
cacheLogger.Warning("补加列失败: " + err.Error())
}
}
} else {
for _, col := range []string{
`ALTER TABLE request_log ADD COLUMN user_agent TEXT`,
`ALTER TABLE request_log ADD COLUMN created_at INTEGER DEFAULT 0`,
`ALTER TABLE request_log ADD COLUMN updated_at INTEGER DEFAULT 0`,
} {
if _, err := cacheDB.Exec(col); err != nil {
cacheLogger.Debug("补列跳过(已存在): " + err.Error())
}
}
}
// append-only 日志表:旧行按 ts 回填两个时间列(幂等)
if _, err := cacheDB.Exec(q(`UPDATE request_log SET created_at = ts, updated_at = ts WHERE created_at = 0`)); err != nil {
cacheLogger.Warning("request_log时间列回填失败: " + err.Error())
}
for _, idx := range []string{
`CREATE INDEX IF NOT EXISTS idx_request_log_ts ON request_log(ts)`,
`CREATE INDEX IF NOT EXISTS idx_request_log_ip ON request_log(ip)`,
`CREATE INDEX IF NOT EXISTS idx_request_log_status ON request_log(status)`,
} {
if _, err := cacheDB.Exec(idx); err != nil {
cacheLogger.Warning("创建request_log索引失败: " + err.Error())
}
} }
reqLogCh = make(chan RequestLogEntry, reqLogChanSize) reqLogCh = make(chan RequestLogEntry, reqLogChanSize)
@@ -109,7 +149,7 @@ func GetRecentRequests(limit int) []RequestLogEntry {
return nil return nil
} }
rows, err := cacheDB.Query( rows, err := cacheDB.Query(
q("SELECT ts, ip, method, path, query, status, duration_ms, COALESCE(error_msg, '') FROM request_log ORDER BY id DESC LIMIT ?"), limit) q("SELECT ts, ip, method, path, query, status, duration_ms, COALESCE(error_msg, ''), COALESCE(user_agent, '') FROM request_log ORDER BY id DESC LIMIT ?"), limit)
if err != nil { if err != nil {
cacheLogger.Warning("查询请求日志失败: " + err.Error()) cacheLogger.Warning("查询请求日志失败: " + err.Error())
return nil return nil
@@ -119,7 +159,7 @@ func GetRecentRequests(limit int) []RequestLogEntry {
entries := make([]RequestLogEntry, 0, limit) entries := make([]RequestLogEntry, 0, limit)
for rows.Next() { for rows.Next() {
var e RequestLogEntry var e RequestLogEntry
if err := rows.Scan(&e.Ts, &e.IP, &e.Method, &e.Path, &e.Query, &e.Status, &e.DurationMs, &e.ErrorMsg); err != nil { if err := rows.Scan(&e.Ts, &e.IP, &e.Method, &e.Path, &e.Query, &e.Status, &e.DurationMs, &e.ErrorMsg, &e.UserAgent); err != nil {
cacheLogger.Warning("读取请求日志失败: " + err.Error()) cacheLogger.Warning("读取请求日志失败: " + err.Error())
break break
} }
@@ -128,6 +168,66 @@ func GetRecentRequests(limit int) []RequestLogEntry {
return entries return entries
} }
// DailyStat 一天的请求量统计
type DailyStat struct {
Day string `json:"day"` // 本地时区 MM-DD
Count int64 `json:"count"`
Pct int `json:"pct"` // 相对窗口峰值的百分比(进度条宽度)
MaxCount bool `json:"max_count"` // 是否为窗口内峰值(前端高亮用)
}
// GetDailyRequestStats 按本地时区分天统计最近 days 天(含今天)的请求量,依赖 request_log。
// ponytail: (ts+固定偏移)/86400 分组,DST 切换日边界误差 1 小时,趋势展示可接受
func GetDailyRequestStats(days int) []DailyStat {
if cacheDB == nil || reqLogCh == nil || days <= 0 {
return nil
}
_, off := time.Now().Zone()
day := func(ts int64) int64 { return (ts + int64(off)) / 86400 }
cutoff := (day(time.Now().Unix()) - int64(days) + 1) * 86400 - int64(off)
rows, err := cacheDB.Query(q(
"SELECT (ts + ?) / 86400, COUNT(*) FROM request_log WHERE ts >= ? GROUP BY 1 ORDER BY 1"),
int64(off), cutoff)
if err != nil {
cacheLogger.Warning("按天聚合请求量失败: " + err.Error())
return nil
}
defer rows.Close()
counts := make(map[int64]int64, days)
var maxCount int64
for rows.Next() {
var d, c int64
if err := rows.Scan(&d, &c); err != nil {
cacheLogger.Warning("读取按天统计失败: " + err.Error())
break
}
counts[d] = c
if c > maxCount {
maxCount = c
}
}
// 补齐无请求的日期,升序返回
stats := make([]DailyStat, 0, days)
for i := int64(days) - 1; i >= 0; i-- {
d := day(time.Now().Unix()) - i
c := counts[d]
var pct int
if maxCount > 0 {
pct = int(c * 100 / maxCount)
}
stats = append(stats, DailyStat{
Day: time.Unix(d*86400-int64(off), 0).Format("01-02"),
Count: c,
Pct: pct,
MaxCount: c > 0 && c == maxCount,
})
}
return stats
}
// reqLogWorker 攒批落库 + 定期清理过期日志 // reqLogWorker 攒批落库 + 定期清理过期日志
func reqLogWorker() { func reqLogWorker() {
defer close(reqLogDone) defer close(reqLogDone)
@@ -174,24 +274,21 @@ func reqLogWorker() {
} }
} }
// insertRequestLogs 事务批量写入 // insertRequestLogs 多值 INSERT 一次往返写入整批(单条语句自带原子性)
// ponytail: 批量上限 50 行 × 9 列 = 450 参数,远低于 SQLite(999)/PG(65535) 占位符上限
func insertRequestLogs(batch []RequestLogEntry) error { func insertRequestLogs(batch []RequestLogEntry) error {
tx, err := cacheDB.Begin() var b strings.Builder
if err != nil { b.WriteString("INSERT INTO request_log (ts, ip, method, path, query, status, duration_ms, error_msg, user_agent) VALUES ")
return err args := make([]interface{}, 0, len(batch)*9)
} for i, e := range batch {
stmt, err := tx.Prepare(q("INSERT INTO request_log (ts, ip, method, path, query, status, duration_ms, error_msg) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")) if i > 0 {
if err != nil { b.WriteByte(',')
tx.Rollback()
return err
}
for _, e := range batch {
if _, err := stmt.Exec(e.Ts, e.IP, e.Method, e.Path, e.Query, e.Status, e.DurationMs, e.ErrorMsg); err != nil {
tx.Rollback()
return err
} }
b.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, e.Ts, e.IP, e.Method, e.Path, e.Query, e.Status, e.DurationMs, e.ErrorMsg, e.UserAgent)
} }
return tx.Commit() _, err := cacheDB.Exec(q(b.String()), args...)
return err
} }
// cleanupRequestLog 删除超过保留期的日志 // cleanupRequestLog 删除超过保留期的日志
+20 -9
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"net" "net"
"net/http" "net/http"
"regexp"
"strings" "strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
@@ -253,26 +252,38 @@ func LogRequestMiddleware(handler http.HandlerFunc) http.HandlerFunc {
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
handler(sw, r) handler(sw, r)
query := sanitizeQuery(r.URL.RawQuery) ua := r.UserAgent()
if len(ua) > 512 { // 防恶意超长 UA 撑爆日志
ua = ua[:512]
}
fqapi.LogRequest(fqapi.RequestLogEntry{ fqapi.LogRequest(fqapi.RequestLogEntry{
Ts: start.Unix(), Ts: start.Unix(),
IP: clientIP(r), IP: clientIP(r),
Method: r.Method, Method: r.Method,
Path: r.URL.Path, Path: r.URL.Path,
Query: query, Query: decodedQuery(r),
Status: sw.status, Status: sw.status,
DurationMs: time.Since(start).Milliseconds(), DurationMs: time.Since(start).Milliseconds(),
ErrorMsg: string(sw.errBuf), ErrorMsg: string(sw.errBuf),
UserAgent: ua,
}) })
} }
} }
// keyParamRe 匹配查询串中独立的 key 参数(accesskey 等不会被误伤) // decodedQuery 重组解码后的查询串(keyword=%E6%B5%8B → keyword=测),日志可读
var keyParamRe = regexp.MustCompile(`(?i)(^|&)key=[^&]*`) // ponytail: map 遍历乱序、%26 解码后与分隔符同名 —— 展示用途可接受
func decodedQuery(r *http.Request) string {
// sanitizeQuery 日志中脱敏:key 参数值替换为 ***(直接替换避免 Encode 把 *** 转义) vals := r.URL.Query() // 已解码
func sanitizeQuery(rawQuery string) string { if len(vals) == 0 {
return keyParamRe.ReplaceAllString(rawQuery, "${1}key=***") return ""
}
parts := make([]string, 0, len(vals))
for k, vs := range vals {
for _, v := range vs {
parts = append(parts, k+"="+v)
}
}
return strings.Join(parts, "&")
} }
// HandleHealth 存活探针(供容器/反向代理使用) // HandleHealth 存活探针(供容器/反向代理使用)
+20
View File
@@ -68,9 +68,17 @@ func HandleStatus(w http.ResponseWriter, r *http.Request) {
"Status": e.Status, "Status": e.Status,
"IsErr": e.Status >= 400, "IsErr": e.Status >= 400,
"Duration": fmt.Sprintf("%dms", e.DurationMs), "Duration": fmt.Sprintf("%dms", e.DurationMs),
"UA": truncateStr(e.UserAgent, 60),
}) })
} }
// 近 7 天请求量(日志未启用时为空);最后一天即今日
var todayCount int64
daily := fqapi.GetDailyRequestStats(7)
if n := len(daily); n > 0 {
todayCount = daily[n-1].Count
}
statusTmpl.Execute(w, map[string]interface{}{ statusTmpl.Execute(w, map[string]interface{}{
"NowTime": time.Now().Format("2006-01-02 15:04:05"), "NowTime": time.Now().Format("2006-01-02 15:04:05"),
"Uptime": uptimeStr, "Uptime": uptimeStr,
@@ -94,6 +102,9 @@ func HandleStatus(w http.ResponseWriter, r *http.Request) {
"StatAudio": stats["audio"], "StatAudio": stats["audio"],
"StatVideo": stats["video"], "StatVideo": stats["video"],
"StatCategory": stats["category"], "StatCategory": stats["category"],
"TodayCount": todayCount,
"DailyStats": daily,
"RateLimit": getRateLimitDesc(),
"CacheCount": getDBCacheCount(), "CacheCount": getDBCacheCount(),
"CacheStatus": getCacheStatus(), "CacheStatus": getCacheStatus(),
"CleanupStatus": getCleanupStatus(), "CleanupStatus": getCleanupStatus(),
@@ -110,6 +121,15 @@ func truncateStr(s string, n int) string {
return s[:n] + "…" return s[:n] + "…"
} }
// getRateLimitDesc 当前每 IP 限流描述(server.rate_limit_qps,0=不限制)
func getRateLimitDesc() string {
cfg := config.GetAppConfig()
if cfg == nil || cfg.Server.RateLimitQPS <= 0 {
return "未启用"
}
return fmt.Sprintf("%.0f QPS/IP", cfg.Server.RateLimitQPS)
}
// getCacheStatus 获取缓存状态 // getCacheStatus 获取缓存状态
func getCacheStatus() string { func getCacheStatus() string {
cfg := config.GetConfig() cfg := config.GetConfig()
+17 -1
View File
@@ -186,6 +186,12 @@
.strack { height: 5px; border-radius: 999px; background: var(--panel-2); overflow: hidden; } .strack { height: 5px; border-radius: 999px; background: var(--panel-2); overflow: hidden; }
.sfill { display: block; height: 100%; border-radius: inherit; background: var(--accent); width: 0; transition: width 0.7s cubic-bezier(0.22, 0.61, 0.36, 1); } .sfill { display: block; height: 100%; border-radius: inherit; background: var(--accent); width: 0; transition: width 0.7s cubic-bezier(0.22, 0.61, 0.36, 1); }
.sval { font-size: 12px; font-weight: 600; color: var(--ink-2); text-align: right; } .sval { font-size: 12px; font-weight: 600; color: var(--ink-2); text-align: right; }
/* 每日请求行:结构同 srow,宽度服务端算好,不参与 .srow 的 JS 归一化 */
.drows { margin-top: 10px; padding-top: 10px; border-top: 1px dashed var(--border); display: flex; flex-direction: column; gap: 7px; }
.drow { display: grid; grid-template-columns: 44px 1fr 52px; gap: 8px; align-items: center; }
.dlabel { font-size: 11.5px; color: var(--ink-3); font-variant-numeric: tabular-nums; }
.dval { font-size: 12px; font-weight: 600; color: var(--ink-2); text-align: right; }
.drow.is-max .dlabel, .drow.is-max .dval { color: var(--accent); }
/* ── 最近请求表 ─────────────────────── */ /* ── 最近请求表 ─────────────────────── */
.req-card { margin-bottom: 14px; overflow: hidden; } .req-card { margin-bottom: 14px; overflow: hidden; }
@@ -472,6 +478,7 @@
<div class="hero-meta"> <div class="hero-meta">
<div class="kv"><span class="k">签名算法</span><span class="v num">{{.Signature}}</span></div> <div class="kv"><span class="k">签名算法</span><span class="v num">{{.Signature}}</span></div>
<div class="kv"><span class="k">启动时间</span><span class="v num">{{.StartTime}}</span></div> <div class="kv"><span class="k">启动时间</span><span class="v num">{{.StartTime}}</span></div>
<div class="kv"><span class="k">限流</span><span class="v num">{{.RateLimit}}</span></div>
</div> </div>
</section> </section>
@@ -513,7 +520,7 @@
<span class="kpi-label">请求统计</span> <span class="kpi-label">请求统计</span>
</div> </div>
<div class="big num" data-count-num="{{.StatTotal}}">{{.StatTotal}}</div> <div class="big num" data-count-num="{{.StatTotal}}">{{.StatTotal}}</div>
<div class="big-sub">累计业务请求</div> <div class="big-sub">累计业务请求 · 今日 <span class="num">{{.TodayCount}}</span> 次</div>
<div class="srows"> <div class="srows">
<div class="srow" data-count="{{.StatSearch}}"><span class="slabel">搜索</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatSearch}}</span></div> <div class="srow" data-count="{{.StatSearch}}"><span class="slabel">搜索</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatSearch}}</span></div>
<div class="srow" data-count="{{.StatDetail}}"><span class="slabel">详情</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatDetail}}</span></div> <div class="srow" data-count="{{.StatDetail}}"><span class="slabel">详情</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatDetail}}</span></div>
@@ -524,6 +531,13 @@
<div class="srow" data-count="{{.StatVideo}}"><span class="slabel">视频</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatVideo}}</span></div> <div class="srow" data-count="{{.StatVideo}}"><span class="slabel">视频</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatVideo}}</span></div>
<div class="srow" data-count="{{.StatCategory}}"><span class="slabel">分类</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatCategory}}</span></div> <div class="srow" data-count="{{.StatCategory}}"><span class="slabel">分类</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatCategory}}</span></div>
</div> </div>
{{if .DailyStats}}
<div class="drows">
{{range .DailyStats}}
<div class="drow{{if .MaxCount}} is-max{{end}}"><span class="dlabel">{{.Day}}</span><span class="strack"><i class="sfill" style="width: {{.Pct}}%"></i></span><span class="dval num">{{.Count}}</span></div>
{{end}}
</div>
{{end}}
</section> </section>
<section class="card kpi"> <section class="card kpi">
@@ -553,6 +567,7 @@
<th>方法</th> <th>方法</th>
<th>路径</th> <th>路径</th>
<th>IP</th> <th>IP</th>
<th>UA</th>
<th>状态</th> <th>状态</th>
<th class="r">耗时</th> <th class="r">耗时</th>
</tr> </tr>
@@ -564,6 +579,7 @@
<td><span class="mth">{{.Method}}</span></td> <td><span class="mth">{{.Method}}</span></td>
<td class="code path-cell" title="{{.Path}}">{{.Path}}{{if .Query}} <span class="q">?{{.Query}}</span>{{end}}</td> <td class="code path-cell" title="{{.Path}}">{{.Path}}{{if .Query}} <span class="q">?{{.Query}}</span>{{end}}</td>
<td class="code">{{.IP}}</td> <td class="code">{{.IP}}</td>
<td class="code" title="{{.UA}}">{{.UA}}</td>
<td><span class="stc {{if .IsErr}}stc-err{{else}}stc-ok{{end}}"><i aria-hidden="true"></i>{{.Status}}</span></td> <td><span class="stc {{if .IsErr}}stc-err{{else}}stc-ok{{end}}"><i aria-hidden="true"></i>{{.Status}}</span></td>
<td class="r num">{{.Duration}}</td> <td class="r num">{{.Duration}}</td>
</tr> </tr>