✨ feat(database): 完善请求日志体系并为全表补充时间字段
- request_log 新增 user_agent 列（含旧表 ALTER 迁移）及 ip/status 索引，记录解码后的明文 query（含 key，不脱敏） - 请求日志批量写入改为多值 INSERT 单次往返，替代事务逐条 Exec - 新增按本地时区分天聚合查询，状态页展示今日请求量与近 7 天请求趋势（峰值日高亮） - 新增 TZ 时区支持：.env 配置 TZ 后日志时间戳与状态页全部时间统一生效，二进制内嵌 tzdata 保证容器可用 - 五张表统一补齐 created_at/updated_at 字段，启动时自动补列并按业务时间回填旧行 - device_pool 的 device_header 增加唯一约束；audio_cache 写入改为 UPSERT 修复同 key 重复报错 - 状态页"最近请求"新增 UA 列与限流状态显示；config.yaml 暴露 rate_limit_qps 配置并默认启用请求日志
This commit is contained in:
chenxin committed
1 parent
1a4997e031
commit
126a96d744
12 files changed
+276
-54
No files matched your search
@@ -1,6 +1,9 @@
|
||||
# 敏感配置模板：复制为 .env 并填入真实值（.env 已被 gitignore，不会提交）
|
||||
# 环境变量优先于 config.yaml 中的对应配置
|
||||
|
||||
# 进程时区（IANA 名称），日志时间戳与 /status 页所有时间均按此显示
|
||||
TZ=Asia/Shanghai
|
||||
|
||||
# 服务访问密钥（/status、/self-info 等端点的鉴权）
|
||||
# FQ_SERVER_ACCESS_KEY=your-access-key
|
||||
|
||||
|
||||
@@ -14,6 +14,9 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
// 嵌入 tzdata，容器内无系统时区库时 TZ 仍可用
|
||||
_ "time/tzdata"
|
||||
)
|
||||
|
||||
// Version / BuildTime 由编译时 -ldflags "-X main.Version=... -X main.BuildTime=..." 注入
|
||||
@@ -26,6 +29,9 @@ var (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// 时区生效须先于一切时间显示（.env 中的 TZ）
|
||||
config.ApplyTimezone()
|
||||
|
||||
logger.Info(fmt.Sprintf("番茄小说API服务 版本=%s 编译时间=%s", Version, BuildTime))
|
||||
|
||||
// 加载配置文件
|
||||
|
||||
+3
-1
@@ -5,10 +5,12 @@ server:
|
||||
port: "40180" # 监听端口
|
||||
host: "0.0.0.0" # 监听地址
|
||||
access_key: "" # 访问密钥，为空则不启用验证（用于 /status）
|
||||
rate_limit_qps: 0 # 每 IP 每秒请求上限（令牌桶，突发 10），0=不限制
|
||||
|
||||
# 请求日志（只记元数据与错误摘要，不存响应体；在 /status 页"最近请求"展示）
|
||||
# 注意：日志时间显示与"近 7 天请求"统计依赖 .env 中的 TZ 时区
|
||||
request_log:
|
||||
enabled: false
|
||||
enabled: true
|
||||
retention_days: 7 # 日志保留天数，自动清理
|
||||
|
||||
# 番茄小说配置
|
||||
|
||||
@@ -3,8 +3,10 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
@@ -121,6 +123,22 @@ func NewConfigManager() *ConfigManager {
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyTimezone 按 TZ 环境变量/.env 设置进程时区。
|
||||
// 必须在任何时间显示之前调用（time.Local 首次使用后不再重读 TZ）。
|
||||
// 环境变量已设 TZ 时以环境变量为准（.env 不覆盖已有环境变量）。
|
||||
func ApplyTimezone() {
|
||||
loadDotenv("./.env")
|
||||
tz := os.Getenv("TZ")
|
||||
if tz == "" {
|
||||
return
|
||||
}
|
||||
if _, err := time.LoadLocation(tz); err != nil {
|
||||
log.Printf("[config/WARN] 无效时区 TZ=%s，沿用系统时区: %v", tz, err)
|
||||
return // 不写入 TZ：无效值会让 time.Local 落到 UTC
|
||||
}
|
||||
os.Setenv("TZ", tz)
|
||||
}
|
||||
|
||||
// LoadConfig 加载配置文件
|
||||
func LoadConfig(configPath string) error {
|
||||
// 初始化管理器
|
||||
|
||||
@@ -38,9 +38,11 @@ func GenerateAudioProxyURL(originalURL, deviceID string, headers map[string]stri
|
||||
if cacheDB != nil {
|
||||
|
||||
_, err := cacheDB.Exec(q(`
|
||||
INSERT INTO audio_cache (key, original_url, device_id, headers, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`), key, originalURL, deviceID, string(headersJSON), time.Now().Unix())
|
||||
INSERT INTO audio_cache (key, original_url, device_id, headers, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET original_url = excluded.original_url,
|
||||
device_id = excluded.device_id, headers = excluded.headers, updated_at = excluded.updated_at
|
||||
`), key, originalURL, deviceID, string(headersJSON), time.Now().Unix(), time.Now().Unix())
|
||||
if err != nil {
|
||||
audioLogger.Error("保存音频缓存失败: " + err.Error())
|
||||
}
|
||||
|
||||
@@ -149,12 +149,14 @@ func createTables() error {
|
||||
`CREATE TABLE IF NOT EXISTS cache (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS cache (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
created_at INTEGER DEFAULT 0
|
||||
created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
|
||||
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
|
||||
)`},
|
||||
{"statistics",
|
||||
`CREATE TABLE IF NOT EXISTS statistics (
|
||||
@@ -168,6 +170,7 @@ func createTables() error {
|
||||
audio_requests INTEGER DEFAULT 0,
|
||||
video_requests INTEGER DEFAULT 0,
|
||||
category_requests INTEGER DEFAULT 0,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS statistics (
|
||||
@@ -181,26 +184,29 @@ func createTables() error {
|
||||
audio_requests INTEGER DEFAULT 0,
|
||||
video_requests INTEGER DEFAULT 0,
|
||||
category_requests INTEGER DEFAULT 0,
|
||||
updated_at INTEGER DEFAULT 0
|
||||
created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
|
||||
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
|
||||
)`},
|
||||
{"device_pool",
|
||||
`CREATE TABLE IF NOT EXISTS device_pool (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
device_header TEXT NOT NULL,
|
||||
device_header TEXT NOT NULL UNIQUE,
|
||||
key TEXT NOT NULL,
|
||||
key_ver INTEGER NOT NULL,
|
||||
is_valid INTEGER DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
last_used_at INTEGER NOT NULL
|
||||
last_used_at INTEGER NOT NULL,
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS device_pool (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
device_header TEXT NOT NULL,
|
||||
device_header TEXT NOT NULL UNIQUE,
|
||||
key TEXT NOT NULL,
|
||||
key_ver INTEGER NOT NULL,
|
||||
is_valid INTEGER DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
last_used_at INTEGER NOT NULL
|
||||
last_used_at INTEGER NOT NULL,
|
||||
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
|
||||
)`},
|
||||
{"audio_cache",
|
||||
`CREATE TABLE IF NOT EXISTS audio_cache (
|
||||
@@ -208,14 +214,16 @@ func createTables() error {
|
||||
original_url TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
headers TEXT,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS audio_cache (
|
||||
key TEXT PRIMARY KEY,
|
||||
original_url TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
headers TEXT,
|
||||
created_at INTEGER DEFAULT 0
|
||||
created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
|
||||
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
|
||||
)`},
|
||||
}
|
||||
|
||||
@@ -242,6 +250,42 @@ func createTables() error {
|
||||
if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`); err != nil {
|
||||
cacheLogger.Warning("创建device_pool索引失败: " + err.Error())
|
||||
}
|
||||
// 旧库补唯一约束：表已存在时 DDL 的 UNIQUE 不会生效，用唯一索引补齐；
|
||||
// 若存量数据已有重复 device_header 则创建失败，仅告警提示人工清理
|
||||
if _, err := cacheDB.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_device_pool_header ON device_pool(device_header)`); err != nil {
|
||||
cacheLogger.Warning("补建device_header唯一索引失败（可能存在重复设备，清理后重启即可）: " + err.Error())
|
||||
}
|
||||
|
||||
// 旧表补 created_at/updated_at 列（新表 DDL 自带，此处仅老库生效）
|
||||
for _, m := range []struct{ table, col, ddl string }{
|
||||
{"cache", "updated_at", "INTEGER DEFAULT 0"}, // SQLite ALTER 不允许非常量默认值，先补 0 再回填
|
||||
{"statistics", "created_at", "INTEGER DEFAULT 0"},
|
||||
{"device_pool", "updated_at", "INTEGER DEFAULT 0"},
|
||||
{"audio_cache", "updated_at", "INTEGER DEFAULT 0"},
|
||||
} {
|
||||
if dbDriver == "pgx" {
|
||||
m.ddl = "BIGINT DEFAULT 0"
|
||||
}
|
||||
alter := "ALTER TABLE " + m.table + " ADD COLUMN "
|
||||
if dbDriver == "pgx" {
|
||||
alter += "IF NOT EXISTS "
|
||||
}
|
||||
if _, err := cacheDB.Exec(alter + m.col + " " + m.ddl); err != nil {
|
||||
cacheLogger.Debug("补列跳过（已存在）: " + m.table + "." + m.col)
|
||||
}
|
||||
}
|
||||
|
||||
// 旧行回填（幂等：仅 0 值行受影响）
|
||||
for _, backfill := range []string{
|
||||
`UPDATE cache SET updated_at = created_at WHERE updated_at = 0`,
|
||||
`UPDATE statistics SET created_at = updated_at WHERE created_at = 0`,
|
||||
`UPDATE device_pool SET updated_at = last_used_at WHERE updated_at = 0`,
|
||||
`UPDATE audio_cache SET updated_at = created_at WHERE updated_at = 0`,
|
||||
} {
|
||||
if _, err := cacheDB.Exec(backfill); err != nil {
|
||||
cacheLogger.Warning("时间列回填失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -337,8 +381,9 @@ func UpdateCache(key string, value string, expi int) error {
|
||||
}
|
||||
|
||||
// UPSERT：ON CONFLICT 语法 SQLite 3.24+ 与 PostgreSQL 通用
|
||||
_, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value`), key, string(jsonData), time.Now().Unix())
|
||||
_, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at, updated_at) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`),
|
||||
key, string(jsonData), time.Now().Unix(), time.Now().Unix())
|
||||
if err != nil {
|
||||
cacheLogger.Error("更新缓存失败: " + err.Error())
|
||||
return err
|
||||
|
||||
@@ -30,7 +30,8 @@ func TestConcurrentCacheAccess(t *testing.T) {
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -73,7 +74,8 @@ func TestWithCache(t *testing.T) {
|
||||
}()
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache (
|
||||
key TEXT PRIMARY KEY, value TEXT,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -401,11 +401,11 @@ func (p *DeviceKeyPool) saveDeviceToDB(item *DeviceKeyItem) error {
|
||||
|
||||
// RETURNING id 同时兼容 SQLite(3.35+) 与 PostgreSQL，替代 LastInsertId()
|
||||
err := cacheDB.QueryRow(q(`
|
||||
INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
RETURNING id
|
||||
`), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid),
|
||||
item.CreatedAt.Unix(), item.LastUsedAt.Unix()).Scan(&item.ID)
|
||||
item.CreatedAt.Unix(), item.LastUsedAt.Unix(), item.CreatedAt.Unix()).Scan(&item.ID)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -418,10 +418,10 @@ func (p *DeviceKeyPool) updateDeviceInDB(item *DeviceKeyItem) error {
|
||||
|
||||
_, err := cacheDB.Exec(q(`
|
||||
UPDATE device_pool
|
||||
SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ?
|
||||
SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
`), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid),
|
||||
item.LastUsedAt.Unix(), item.ID)
|
||||
item.LastUsedAt.Unix(), item.LastUsedAt.Unix(), item.ID)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
+119
-22
@@ -3,6 +3,7 @@ package fqapi
|
||||
import (
|
||||
"fanqie/internal/config"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -15,10 +16,11 @@ type RequestLogEntry struct {
|
||||
IP string `json:"ip"`
|
||||
Method string `json:"method"`
|
||||
Path string `json:"path"`
|
||||
Query string `json:"query"` // 已脱敏（key 参数替换为 ***）
|
||||
Query string `json:"query"` // 解码后查询串（含 key，不脱敏）
|
||||
Status int `json:"status"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
ErrorMsg string `json:"error_msg"` // 仅 status>=400 时截断保留
|
||||
UserAgent string `json:"user_agent"` // 客户端 UA，排障时区分调用方
|
||||
}
|
||||
|
||||
// ErrMsgMax 错误响应摘要的最大保留字节数
|
||||
@@ -52,7 +54,10 @@ func InitRequestLog() error {
|
||||
query TEXT,
|
||||
status INTEGER,
|
||||
duration_ms INTEGER,
|
||||
error_msg TEXT
|
||||
error_msg TEXT,
|
||||
user_agent TEXT,
|
||||
created_at INTEGER DEFAULT (strftime('%s', 'now')),
|
||||
updated_at INTEGER DEFAULT (strftime('%s', 'now'))
|
||||
)`
|
||||
if dbDriver == "pgx" {
|
||||
ddl = `CREATE TABLE IF NOT EXISTS request_log (
|
||||
@@ -64,15 +69,50 @@ func InitRequestLog() error {
|
||||
query TEXT,
|
||||
status INTEGER,
|
||||
duration_ms BIGINT,
|
||||
error_msg TEXT
|
||||
error_msg TEXT,
|
||||
user_agent TEXT,
|
||||
created_at BIGINT DEFAULT extract(epoch FROM now())::bigint,
|
||||
updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint
|
||||
)`
|
||||
}
|
||||
if _, err := cacheDB.Exec(ddl); err != nil {
|
||||
return fmt.Errorf("创建request_log表失败: %w", err)
|
||||
}
|
||||
if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_request_log_ts ON request_log(ts)`); err != nil {
|
||||
// 旧表补列（表已存在时 CREATE 不会加列；PG 支持 IF NOT EXISTS，SQLite 报错即视为已存在）
|
||||
if dbDriver == "pgx" {
|
||||
for _, col := range []string{
|
||||
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS user_agent TEXT`,
|
||||
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS created_at BIGINT DEFAULT 0`,
|
||||
`ALTER TABLE request_log ADD COLUMN IF NOT EXISTS updated_at BIGINT DEFAULT 0`,
|
||||
} {
|
||||
if _, err := cacheDB.Exec(col); err != nil {
|
||||
cacheLogger.Warning("补加列失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, col := range []string{
|
||||
`ALTER TABLE request_log ADD COLUMN user_agent TEXT`,
|
||||
`ALTER TABLE request_log ADD COLUMN created_at INTEGER DEFAULT 0`,
|
||||
`ALTER TABLE request_log ADD COLUMN updated_at INTEGER DEFAULT 0`,
|
||||
} {
|
||||
if _, err := cacheDB.Exec(col); err != nil {
|
||||
cacheLogger.Debug("补列跳过（已存在）: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
// append-only 日志表：旧行按 ts 回填两个时间列（幂等）
|
||||
if _, err := cacheDB.Exec(q(`UPDATE request_log SET created_at = ts, updated_at = ts WHERE created_at = 0`)); err != nil {
|
||||
cacheLogger.Warning("request_log时间列回填失败: " + err.Error())
|
||||
}
|
||||
for _, idx := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_request_log_ts ON request_log(ts)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_request_log_ip ON request_log(ip)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_request_log_status ON request_log(status)`,
|
||||
} {
|
||||
if _, err := cacheDB.Exec(idx); err != nil {
|
||||
cacheLogger.Warning("创建request_log索引失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
reqLogCh = make(chan RequestLogEntry, reqLogChanSize)
|
||||
reqLogQuit = make(chan struct{})
|
||||
@@ -109,7 +149,7 @@ func GetRecentRequests(limit int) []RequestLogEntry {
|
||||
return nil
|
||||
}
|
||||
rows, err := cacheDB.Query(
|
||||
q("SELECT ts, ip, method, path, query, status, duration_ms, COALESCE(error_msg, '') FROM request_log ORDER BY id DESC LIMIT ?"), limit)
|
||||
q("SELECT ts, ip, method, path, query, status, duration_ms, COALESCE(error_msg, ''), COALESCE(user_agent, '') FROM request_log ORDER BY id DESC LIMIT ?"), limit)
|
||||
if err != nil {
|
||||
cacheLogger.Warning("查询请求日志失败: " + err.Error())
|
||||
return nil
|
||||
@@ -119,7 +159,7 @@ func GetRecentRequests(limit int) []RequestLogEntry {
|
||||
entries := make([]RequestLogEntry, 0, limit)
|
||||
for rows.Next() {
|
||||
var e RequestLogEntry
|
||||
if err := rows.Scan(&e.Ts, &e.IP, &e.Method, &e.Path, &e.Query, &e.Status, &e.DurationMs, &e.ErrorMsg); err != nil {
|
||||
if err := rows.Scan(&e.Ts, &e.IP, &e.Method, &e.Path, &e.Query, &e.Status, &e.DurationMs, &e.ErrorMsg, &e.UserAgent); err != nil {
|
||||
cacheLogger.Warning("读取请求日志失败: " + err.Error())
|
||||
break
|
||||
}
|
||||
@@ -128,6 +168,66 @@ func GetRecentRequests(limit int) []RequestLogEntry {
|
||||
return entries
|
||||
}
|
||||
|
||||
// DailyStat 一天的请求量统计
|
||||
type DailyStat struct {
|
||||
Day string `json:"day"` // 本地时区 MM-DD
|
||||
Count int64 `json:"count"`
|
||||
Pct int `json:"pct"` // 相对窗口峰值的百分比（进度条宽度）
|
||||
MaxCount bool `json:"max_count"` // 是否为窗口内峰值（前端高亮用）
|
||||
}
|
||||
|
||||
// GetDailyRequestStats 按本地时区分天统计最近 days 天（含今天）的请求量，依赖 request_log。
|
||||
// ponytail: (ts+固定偏移)/86400 分组，DST 切换日边界误差 1 小时，趋势展示可接受
|
||||
func GetDailyRequestStats(days int) []DailyStat {
|
||||
if cacheDB == nil || reqLogCh == nil || days <= 0 {
|
||||
return nil
|
||||
}
|
||||
_, off := time.Now().Zone()
|
||||
day := func(ts int64) int64 { return (ts + int64(off)) / 86400 }
|
||||
cutoff := (day(time.Now().Unix()) - int64(days) + 1) * 86400 - int64(off)
|
||||
|
||||
rows, err := cacheDB.Query(q(
|
||||
"SELECT (ts + ?) / 86400, COUNT(*) FROM request_log WHERE ts >= ? GROUP BY 1 ORDER BY 1"),
|
||||
int64(off), cutoff)
|
||||
if err != nil {
|
||||
cacheLogger.Warning("按天聚合请求量失败: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
counts := make(map[int64]int64, days)
|
||||
var maxCount int64
|
||||
for rows.Next() {
|
||||
var d, c int64
|
||||
if err := rows.Scan(&d, &c); err != nil {
|
||||
cacheLogger.Warning("读取按天统计失败: " + err.Error())
|
||||
break
|
||||
}
|
||||
counts[d] = c
|
||||
if c > maxCount {
|
||||
maxCount = c
|
||||
}
|
||||
}
|
||||
|
||||
// 补齐无请求的日期，升序返回
|
||||
stats := make([]DailyStat, 0, days)
|
||||
for i := int64(days) - 1; i >= 0; i-- {
|
||||
d := day(time.Now().Unix()) - i
|
||||
c := counts[d]
|
||||
var pct int
|
||||
if maxCount > 0 {
|
||||
pct = int(c * 100 / maxCount)
|
||||
}
|
||||
stats = append(stats, DailyStat{
|
||||
Day: time.Unix(d*86400-int64(off), 0).Format("01-02"),
|
||||
Count: c,
|
||||
Pct: pct,
|
||||
MaxCount: c > 0 && c == maxCount,
|
||||
})
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
// reqLogWorker 攒批落库 + 定期清理过期日志
|
||||
func reqLogWorker() {
|
||||
defer close(reqLogDone)
|
||||
@@ -174,24 +274,21 @@ func reqLogWorker() {
|
||||
}
|
||||
}
|
||||
|
||||
// insertRequestLogs 事务批量写入
|
||||
// insertRequestLogs 多值 INSERT 一次往返写入整批（单条语句自带原子性）
|
||||
// ponytail: 批量上限 50 行 × 9 列 = 450 参数，远低于 SQLite(999)/PG(65535) 占位符上限
|
||||
func insertRequestLogs(batch []RequestLogEntry) error {
|
||||
tx, err := cacheDB.Begin()
|
||||
if err != nil {
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO request_log (ts, ip, method, path, query, status, duration_ms, error_msg, user_agent) VALUES ")
|
||||
args := make([]interface{}, 0, len(batch)*9)
|
||||
for i, e := range batch {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
args = append(args, e.Ts, e.IP, e.Method, e.Path, e.Query, e.Status, e.DurationMs, e.ErrorMsg, e.UserAgent)
|
||||
}
|
||||
_, err := cacheDB.Exec(q(b.String()), args...)
|
||||
return err
|
||||
}
|
||||
stmt, err := tx.Prepare(q("INSERT INTO request_log (ts, ip, method, path, query, status, duration_ms, error_msg) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"))
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
for _, e := range batch {
|
||||
if _, err := stmt.Exec(e.Ts, e.IP, e.Method, e.Path, e.Query, e.Status, e.DurationMs, e.ErrorMsg); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// cleanupRequestLog 删除超过保留期的日志
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -253,26 +252,38 @@ func LogRequestMiddleware(handler http.HandlerFunc) http.HandlerFunc {
|
||||
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
handler(sw, r)
|
||||
|
||||
query := sanitizeQuery(r.URL.RawQuery)
|
||||
ua := r.UserAgent()
|
||||
if len(ua) > 512 { // 防恶意超长 UA 撑爆日志
|
||||
ua = ua[:512]
|
||||
}
|
||||
fqapi.LogRequest(fqapi.RequestLogEntry{
|
||||
Ts: start.Unix(),
|
||||
IP: clientIP(r),
|
||||
Method: r.Method,
|
||||
Path: r.URL.Path,
|
||||
Query: query,
|
||||
Query: decodedQuery(r),
|
||||
Status: sw.status,
|
||||
DurationMs: time.Since(start).Milliseconds(),
|
||||
ErrorMsg: string(sw.errBuf),
|
||||
UserAgent: ua,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// keyParamRe 匹配查询串中独立的 key 参数（accesskey 等不会被误伤）
|
||||
var keyParamRe = regexp.MustCompile(`(?i)(^|&)key=[^&]*`)
|
||||
|
||||
// sanitizeQuery 日志中脱敏：key 参数值替换为 ***（直接替换避免 Encode 把 *** 转义）
|
||||
func sanitizeQuery(rawQuery string) string {
|
||||
return keyParamRe.ReplaceAllString(rawQuery, "${1}key=***")
|
||||
// decodedQuery 重组解码后的查询串（keyword=%E6%B5%8B → keyword=测），日志可读
|
||||
// ponytail: map 遍历乱序、%26 解码后与分隔符同名 —— 展示用途可接受
|
||||
func decodedQuery(r *http.Request) string {
|
||||
vals := r.URL.Query() // 已解码
|
||||
if len(vals) == 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(vals))
|
||||
for k, vs := range vals {
|
||||
for _, v := range vs {
|
||||
parts = append(parts, k+"="+v)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "&")
|
||||
}
|
||||
|
||||
// HandleHealth 存活探针（供容器/反向代理使用）
|
||||
|
||||
@@ -68,9 +68,17 @@ func HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"Status": e.Status,
|
||||
"IsErr": e.Status >= 400,
|
||||
"Duration": fmt.Sprintf("%dms", e.DurationMs),
|
||||
"UA": truncateStr(e.UserAgent, 60),
|
||||
})
|
||||
}
|
||||
|
||||
// 近 7 天请求量（日志未启用时为空）；最后一天即今日
|
||||
var todayCount int64
|
||||
daily := fqapi.GetDailyRequestStats(7)
|
||||
if n := len(daily); n > 0 {
|
||||
todayCount = daily[n-1].Count
|
||||
}
|
||||
|
||||
statusTmpl.Execute(w, map[string]interface{}{
|
||||
"NowTime": time.Now().Format("2006-01-02 15:04:05"),
|
||||
"Uptime": uptimeStr,
|
||||
@@ -94,6 +102,9 @@ func HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"StatAudio": stats["audio"],
|
||||
"StatVideo": stats["video"],
|
||||
"StatCategory": stats["category"],
|
||||
"TodayCount": todayCount,
|
||||
"DailyStats": daily,
|
||||
"RateLimit": getRateLimitDesc(),
|
||||
"CacheCount": getDBCacheCount(),
|
||||
"CacheStatus": getCacheStatus(),
|
||||
"CleanupStatus": getCleanupStatus(),
|
||||
@@ -110,6 +121,15 @@ func truncateStr(s string, n int) string {
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// getRateLimitDesc 当前每 IP 限流描述（server.rate_limit_qps，0=不限制）
|
||||
func getRateLimitDesc() string {
|
||||
cfg := config.GetAppConfig()
|
||||
if cfg == nil || cfg.Server.RateLimitQPS <= 0 {
|
||||
return "未启用"
|
||||
}
|
||||
return fmt.Sprintf("%.0f QPS/IP", cfg.Server.RateLimitQPS)
|
||||
}
|
||||
|
||||
// getCacheStatus 获取缓存状态
|
||||
func getCacheStatus() string {
|
||||
cfg := config.GetConfig()
|
||||
|
||||
@@ -186,6 +186,12 @@
|
||||
.strack { height: 5px; border-radius: 999px; background: var(--panel-2); overflow: hidden; }
|
||||
.sfill { display: block; height: 100%; border-radius: inherit; background: var(--accent); width: 0; transition: width 0.7s cubic-bezier(0.22, 0.61, 0.36, 1); }
|
||||
.sval { font-size: 12px; font-weight: 600; color: var(--ink-2); text-align: right; }
|
||||
/* 每日请求行：结构同 srow，宽度服务端算好，不参与 .srow 的 JS 归一化 */
|
||||
.drows { margin-top: 10px; padding-top: 10px; border-top: 1px dashed var(--border); display: flex; flex-direction: column; gap: 7px; }
|
||||
.drow { display: grid; grid-template-columns: 44px 1fr 52px; gap: 8px; align-items: center; }
|
||||
.dlabel { font-size: 11.5px; color: var(--ink-3); font-variant-numeric: tabular-nums; }
|
||||
.dval { font-size: 12px; font-weight: 600; color: var(--ink-2); text-align: right; }
|
||||
.drow.is-max .dlabel, .drow.is-max .dval { color: var(--accent); }
|
||||
|
||||
/* ── 最近请求表 ─────────────────────── */
|
||||
.req-card { margin-bottom: 14px; overflow: hidden; }
|
||||
@@ -472,6 +478,7 @@
|
||||
<div class="hero-meta">
|
||||
<div class="kv"><span class="k">签名算法</span><span class="v num">{{.Signature}}</span></div>
|
||||
<div class="kv"><span class="k">启动时间</span><span class="v num">{{.StartTime}}</span></div>
|
||||
<div class="kv"><span class="k">限流</span><span class="v num">{{.RateLimit}}</span></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -513,7 +520,7 @@
|
||||
<span class="kpi-label">请求统计</span>
|
||||
</div>
|
||||
<div class="big num" data-count-num="{{.StatTotal}}">{{.StatTotal}}</div>
|
||||
<div class="big-sub">累计业务请求</div>
|
||||
<div class="big-sub">累计业务请求 · 今日 <span class="num">{{.TodayCount}}</span> 次</div>
|
||||
<div class="srows">
|
||||
<div class="srow" data-count="{{.StatSearch}}"><span class="slabel">搜索</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatSearch}}</span></div>
|
||||
<div class="srow" data-count="{{.StatDetail}}"><span class="slabel">详情</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatDetail}}</span></div>
|
||||
@@ -524,6 +531,13 @@
|
||||
<div class="srow" data-count="{{.StatVideo}}"><span class="slabel">视频</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatVideo}}</span></div>
|
||||
<div class="srow" data-count="{{.StatCategory}}"><span class="slabel">分类</span><span class="strack"><i class="sfill"></i></span><span class="sval num">{{.StatCategory}}</span></div>
|
||||
</div>
|
||||
{{if .DailyStats}}
|
||||
<div class="drows">
|
||||
{{range .DailyStats}}
|
||||
<div class="drow{{if .MaxCount}} is-max{{end}}"><span class="dlabel">{{.Day}}</span><span class="strack"><i class="sfill" style="width: {{.Pct}}%"></i></span><span class="dval num">{{.Count}}</span></div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
<section class="card kpi">
|
||||
@@ -553,6 +567,7 @@
|
||||
<th>方法</th>
|
||||
<th>路径</th>
|
||||
<th>IP</th>
|
||||
<th>UA</th>
|
||||
<th>状态</th>
|
||||
<th class="r">耗时</th>
|
||||
</tr>
|
||||
@@ -564,6 +579,7 @@
|
||||
<td><span class="mth">{{.Method}}</span></td>
|
||||
<td class="code path-cell" title="{{.Path}}">{{.Path}}{{if .Query}} <span class="q">?{{.Query}}</span>{{end}}</td>
|
||||
<td class="code">{{.IP}}</td>
|
||||
<td class="code" title="{{.UA}}">{{.UA}}</td>
|
||||
<td><span class="stc {{if .IsErr}}stc-err{{else}}stc-ok{{end}}"><i aria-hidden="true"></i>{{.Status}}</span></td>
|
||||
<td class="r num">{{.Duration}}</td>
|
||||
</tr>
|
||||
|
||||
Reference in new issue
Block a user