From cf16c37f32a69051f33188aa7a12ab75f89858ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=AD=86?= <2685475410@qq.com> Date: Tue, 29 Sep 2026 20:20:17 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(cache):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=20Redis=20=E7=BC=93=E5=AD=98=E4=B8=8E=20PostgreSQL=20=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 cache.redis 配置(独立 host/port/password/db 字段),启用后正文缓存与音频代理缓存走 Redis,键前缀 fq:cache:/fq:audio:,TTL 自动过期 - 新增 cache.backend: sqlite|postgres 切换,PG 连接用 net/url 构造 DSN,密码特殊字符自动转义;SQLite 顺手启用 WAL 与 busy_timeout - SQL 方言收敛:q() 将 ? 占位符转换为 $n、INSERT OR IGNORE 与查询再插入统一为 ON CONFLICT、LastInsertId 改为 RETURNING id、建表 DDL 双版本分派 - PG 下 VACUUM 走 autovacuum 说明、状态页大小改查 pg_database_size - 修复 PG 下设备保存失败:is_valid 列为 INTEGER,写入点补 boolToInt 转换 - 新增 .env 支持:敏感配置(密码/主机/访问密钥)由环境变量覆盖 config.yaml,.env 进 gitignore,仓库附 .env.example 模板 - 删除死代码 InitDevicePoolTable;新增 rebind_test.go 单测 - 更新 README 与 config.yaml 配置示例,go.mod 新增 go-redis/v9 与 pgx/v5 依赖 --- .env.example | 15 ++ .gitignore | 7 +- README.md | 55 ++++- cmd/fanqie/main.go | 7 + config/config.yaml | 62 ++++-- go.mod | 12 +- go.sum | 32 ++- internal/config/config.go | 51 ++++- internal/config/dotenv.go | 49 +++++ internal/fqapi/audio_proxy.go | 97 +++++---- internal/fqapi/cache_manager.go | 326 +++++++++++++++++++++--------- internal/fqapi/device_key_pool.go | 63 ++---- internal/fqapi/rebind_test.go | 20 ++ internal/fqapi/store_redis.go | 85 ++++++++ 14 files changed, 671 insertions(+), 210 deletions(-) create mode 100644 .env.example create mode 100644 internal/config/dotenv.go create mode 100644 internal/fqapi/rebind_test.go create mode 100644 internal/fqapi/store_redis.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7ac66d6 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# 敏感配置模板：复制为 .env 并填入真实值（.env 已被 gitignore，不会提交） +# 环境变量优先于 config.yaml 中的对应配置 + +# 服务访问密钥（/status、/self-info 等端点的鉴权） +# FQ_SERVER_ACCESS_KEY=your-access-key + +# PostgreSQL 连接（覆盖 cache.postgres.*） +FQ_PG_HOST=127.0.0.1 +FQ_PG_USER=postgres +FQ_PG_PASSWORD= +FQ_PG_DATABASE=fanqie + +# Redis 连接（覆盖 cache.redis.*） +FQ_REDIS_HOST=127.0.0.1 +FQ_REDIS_PASSWORD= diff --git a/.gitignore b/.gitignore index fc906de..71ca7e2 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,9 @@ __pycache__ # Build output /dist/ -/build/ \ No newline at end of file +/build/ + +cache.db + +# 环境变量（含真实密码，勿提交） +.env diff --git a/README.md b/README.md index 7575fc3..8553cf1 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Go 语言实现的番茄小说非官方 API 服务，完整实现了签名算法 - **签名算法** - Gorgon、8404、Argus、Ladon、Khronos - **设备密钥池** - 多设备轮换、自动补充 -- **SQLite 缓存** - 本地缓存、过期清理、数据库压缩 +- **SQLite 缓存** - 本地缓存、过期清理、数据库压缩；支持切换 PostgreSQL 与 Redis - **配置热更新** - 修改配置无需重启 - **访问控制** - 支持 AccessKey 验证 @@ -266,7 +266,22 @@ fanqie: # 缓存配置 cache: enabled: true # 启用缓存 - sqlite_path: "./cache.db" # SQLite 路径 + backend: sqlite # 存储后端: sqlite | postgres + sqlite_path: "./cache.db" # SQLite 路径（backend 为 sqlite 时使用） + # PostgreSQL连接配置（backend 为 postgres 时使用） + postgres: + host: "127.0.0.1" # 主机 + port: 5432 # 端口 + user: "postgres" # 用户名 + password: "" # 密码 + database: "fanqie" # 数据库名 + sslmode: "disable" # SSL 模式: disable | prefer | require + redis: # Redis 缓存（正文缓存与音频代理缓存，统计数据/设备池仍在 SQL 库） + enabled: false + host: "127.0.0.1" # 主机 + port: 6379 # 端口 + password: "" # 密码 + db: 0 # 数据库编号 cleanup_enabled: true # 启用自动清理 cleanup_seconds: 86400 # 清理间隔（秒） @@ -278,6 +293,28 @@ device_key_pool: max_size: 30 # 最大设备数 ``` +### 敏感配置（.env） + +密码等敏感值不建议写进 `config.yaml`（会进 git 历史）。复制 `.env.example` 为 `.env` 填入真实值即可，`.env` 已被 gitignore： + +```bash +cp .env.example .env +``` + +支持的环境变量（优先于 `config.yaml` 对应项）： + +| 环境变量 | 覆盖配置 | 说明 | +| -------- | -------- | ---- | +| `FQ_SERVER_ACCESS_KEY` | `server.access_key` | 服务访问密钥 | +| `FQ_PG_HOST` | `cache.postgres.host` | PostgreSQL 主机 | +| `FQ_PG_USER` | `cache.postgres.user` | PostgreSQL 用户名 | +| `FQ_PG_PASSWORD` | `cache.postgres.password` | PostgreSQL 密码 | +| `FQ_PG_DATABASE` | `cache.postgres.database` | PostgreSQL 数据库名 | +| `FQ_REDIS_HOST` | `cache.redis.host` | Redis 主机 | +| `FQ_REDIS_PASSWORD` | `cache.redis.password` | Redis 密码 | + +已有环境变量优先于 `.env` 文件中的值。 + ### 配置说明表 | 配置项 | 说明 | 默认值 | @@ -289,8 +326,20 @@ device_key_pool: | `fanqie.signature` | 签名方法 `sign2` / `sign4` | `sign4` | | `fanqie.content.*` | 正文内容处理选项 | 见上文 | | `fanqie.manga.output_mode` | 漫画输出模式 | `url` | -| `cache.enabled` | 是否启用 SQLite 缓存 | `true` | +| `cache.enabled` | 是否启用正文内容缓存 | `true` | +| `cache.backend` | SQL 存储后端 `sqlite` / `postgres` | `sqlite` | | `cache.sqlite_path` | SQLite 数据库路径 | `./cache.db` | +| `cache.postgres.host` | PostgreSQL 主机 | `127.0.0.1` | +| `cache.postgres.port` | PostgreSQL 端口 | `5432` | +| `cache.postgres.user` | PostgreSQL 用户名 | `postgres` | +| `cache.postgres.password` | PostgreSQL 密码 | 空 | +| `cache.postgres.database` | PostgreSQL 数据库名 | `fanqie` | +| `cache.postgres.sslmode` | SSL 模式 | `disable` | +| `cache.redis.enabled` | 启用 Redis 缓存（正文与音频代理缓存走 Redis） | `false` | +| `cache.redis.host` | Redis 主机 | `127.0.0.1` | +| `cache.redis.port` | Redis 端口 | `6379` | +| `cache.redis.password` | Redis 密码 | 空 | +| `cache.redis.db` | Redis 数据库编号 | `0` | | `device_key_pool.enabled` | 是否启用设备密钥池 | `true` | | `device_key_pool.initial_size` | 初始设备数量 | `20` | | `device_key_pool.min_size` | 最小设备数量 | `10` | diff --git a/cmd/fanqie/main.go b/cmd/fanqie/main.go index 225b5ad..9e78bfd 100644 --- a/cmd/fanqie/main.go +++ b/cmd/fanqie/main.go @@ -33,6 +33,13 @@ func main() { os.Exit(1) } defer fqapi.CloseCache() // 程序退出时关闭数据库 + + // 初始化 Redis 缓存（可选） + if err := fqapi.InitRedis(); err != nil { + logger.Error("Redis 初始化失败: " + err.Error()) + os.Exit(1) + } + defer fqapi.CloseRedis() // 加载统计数据 handlers.LoadStatistics() diff --git a/config/config.yaml b/config/config.yaml index 9d3dcda..c192df3 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,9 +2,9 @@ # 服务器配置 server: - port: "40180" # 监听端口 - host: "0.0.0.0" # 监听地址 - access_key: "" # 访问密钥，为空则不启用验证（用于 /status 和 /self-info） + port: "40180" # 监听端口 + host: "0.0.0.0" # 监听地址 + access_key: "" # 访问密钥，为空则不启用验证（用于 /status 和 /self-info） # 番茄小说配置 fanqie: @@ -12,38 +12,38 @@ fanqie: # gorgon: Gorgon算法 # 8404: 备用算法 algorithm: "gorgon" - + # 签名方法: "sign2" 或 "sign4" # sign2: 仅Gorgon签名 # sign4: X-Gorgon + X-Ladon + X-Argus 完整签名 signature: "sign4" - + # 内容处理配置 content: # 是否移除图片（默认false，保留图片） remove_images: false - + # 是否移除图片描述（默认false，保留描述） remove_image_desc: true - + # 是否移除空行（默认true，移除空行） remove_blank_lines: true - + # 是否移除HTML标签（默认false，保留HTML） remove_html_tags: true - + # 是否移除章节名（默认false，保留章节名） remove_chapter_title: true - + # 是否添加段落缩进（默认true，添加两个全角空格） add_paragraph_indent: false - + # 段落间使用双换行（true=\n\n, false=\n） double_newline: false - + # 是否启用段评功能（默认true，启用段评） enable_review: true - + # 漫画处理配置 manga: # 输出模式：proxy / data_uri / url @@ -56,13 +56,34 @@ fanqie: cache: # 是否启用正文内容缓存 enabled: true - - # SQLite数据库文件路径 + + # 存储后端：sqlite | postgres + backend: postgres + + # SQLite数据库文件路径（backend 为 sqlite 时使用） sqlite_path: "./cache.db" - + + # PostgreSQL连接配置（backend 为 postgres 时使用） + # 敏感值（密码等）建议放 .env，见 .env.example；环境变量优先于此处配置 + postgres: + host: "127.0.0.1" + port: 5432 + user: "postgres" + password: "" + database: "fanqie" + sslmode: "disable" # disable | prefer | require + + # Redis缓存（启用后正文缓存与音频代理缓存走 Redis，统计数据与设备池仍在 SQL 库） + redis: + enabled: true + host: "127.0.0.1" + port: 6379 + password: "" + db: 1 + # 是否启用正文内容缓存自动清理 cleanup_enabled: true - + # 正文内容缓存清理时间（秒），0表示不自动清理 cleanup_seconds: 86400 @@ -70,13 +91,12 @@ cache: device_key_pool: # 是否启用设备密钥池 enabled: true - + # 初始化设备数量 initial_size: 20 - + # 最小设备数量（低于此数量自动补充） min_size: 10 - + # 最大设备数量 max_size: 30 - diff --git a/go.mod b/go.mod index 9177164..35bd25d 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,25 @@ module fanqie -go 1.24.0 +go 1.25.0 require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/fsnotify/fsnotify v1.7.0 github.com/google/uuid v1.4.0 + github.com/jackc/pgx/v5 v5.11.0 github.com/mattn/go-sqlite3 v1.14.18 + github.com/redis/go-redis/v9 v9.22.0 github.com/spf13/viper v1.18.2 ) require ( github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect @@ -23,12 +29,12 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.46.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 523e8cb..38b7c6b 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,12 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -16,6 +22,16 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -31,6 +47,8 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -49,13 +67,16 @@ github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -72,6 +93,8 @@ golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -96,9 +119,10 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index 9fbff56..779628b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,10 +51,32 @@ type MangaConfig struct { // CacheConfig 缓存配置 type CacheConfig struct { - Enabled bool `mapstructure:"enabled"` - SQLitePath string `mapstructure:"sqlite_path"` - CleanupEnabled bool `mapstructure:"cleanup_enabled"` // 是否启用缓存清理 - CleanupSeconds int `mapstructure:"cleanup_seconds"` // 缓存清理时间（秒） + Enabled bool `mapstructure:"enabled"` + Backend string `mapstructure:"backend"` // 存储后端: sqlite | postgres + SQLitePath string `mapstructure:"sqlite_path"` + Postgres PostgresConfig `mapstructure:"postgres"` + Redis RedisConfig `mapstructure:"redis"` + CleanupEnabled bool `mapstructure:"cleanup_enabled"` // 是否启用缓存清理 + CleanupSeconds int `mapstructure:"cleanup_seconds"` // 缓存清理时间（秒） +} + +// PostgresConfig PostgreSQL 连接配置 +type PostgresConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + SSLMode string `mapstructure:"sslmode"` // disable | prefer | require +} + +// RedisConfig Redis 缓存配置（启用后正文缓存与音频代理缓存走 Redis，统计数据与设备池仍在 SQL 库） +type RedisConfig struct { + Enabled bool `mapstructure:"enabled"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` } // DeviceKeyPoolConfig 设备密钥池配置 @@ -98,6 +120,9 @@ func LoadConfig(configPath string) error { manager.configPath = configPath } + // 加载 .env（若存在），敏感配置可由环境变量覆盖 + loadDotenv("./.env") + viper.SetConfigFile(configPath) viper.SetConfigType("yaml") @@ -112,6 +137,7 @@ func LoadConfig(configPath string) error { if err := viper.Unmarshal(newConfig); err != nil { return fmt.Errorf("解析配置文件失败: %w", err) } + applyEnvOverrides(newConfig) // 更新配置 appConfig.Store(newConfig) @@ -144,10 +170,26 @@ func setDefaults() { // 缓存配置 viper.SetDefault("cache.enabled", true) + viper.SetDefault("cache.backend", "sqlite") viper.SetDefault("cache.sqlite_path", "./cache.db") viper.SetDefault("cache.cleanup_enabled", false) viper.SetDefault("cache.cleanup_seconds", 86400) + // PostgreSQL 配置 + viper.SetDefault("cache.postgres.host", "127.0.0.1") + viper.SetDefault("cache.postgres.port", 5432) + viper.SetDefault("cache.postgres.user", "postgres") + viper.SetDefault("cache.postgres.password", "") + viper.SetDefault("cache.postgres.database", "fanqie") + viper.SetDefault("cache.postgres.sslmode", "disable") + + // Redis 配置 + viper.SetDefault("cache.redis.enabled", false) + viper.SetDefault("cache.redis.host", "127.0.0.1") + viper.SetDefault("cache.redis.port", 6379) + viper.SetDefault("cache.redis.password", "") + viper.SetDefault("cache.redis.db", 0) + // 设备密钥池配置 viper.SetDefault("device_key_pool.enabled", true) viper.SetDefault("device_key_pool.initial_size", 3) @@ -188,6 +230,7 @@ func ReloadConfig() error { if err := viper.Unmarshal(newConfig); err != nil { return fmt.Errorf("重新解析配置文件失败: %w", err) } + applyEnvOverrides(newConfig) // 更新配置 manager.mu.Lock() diff --git a/internal/config/dotenv.go b/internal/config/dotenv.go new file mode 100644 index 0000000..648c67c --- /dev/null +++ b/internal/config/dotenv.go @@ -0,0 +1,49 @@ +package config + +import ( + "os" + "strings" +) + +// loadDotenv 加载 .env 文件到环境变量（文件不存在则跳过；不覆盖已有环境变量） +func loadDotenv(path string) { + data, err := os.ReadFile(path) + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + k = strings.TrimSpace(k) + v = strings.Trim(strings.TrimSpace(v), `"'`) + if os.Getenv(k) == "" { + os.Setenv(k, v) + } + } +} + +// applyEnvOverrides 用环境变量覆盖敏感配置（配合 .env 使用，真实密码不进 config.yaml） +func applyEnvOverrides(c *Config) { + for _, o := range []struct { + env string + dst *string + }{ + {"FQ_SERVER_ACCESS_KEY", &c.Server.AccessKey}, + {"FQ_PG_HOST", &c.Cache.Postgres.Host}, + {"FQ_PG_USER", &c.Cache.Postgres.User}, + {"FQ_PG_PASSWORD", &c.Cache.Postgres.Password}, + {"FQ_PG_DATABASE", &c.Cache.Postgres.Database}, + {"FQ_REDIS_HOST", &c.Cache.Redis.Host}, + {"FQ_REDIS_PASSWORD", &c.Cache.Redis.Password}, + } { + if v := os.Getenv(o.env); v != "" { + *o.dst = v + } + } +} diff --git a/internal/fqapi/audio_proxy.go b/internal/fqapi/audio_proxy.go index 4645914..73a34d9 100644 --- a/internal/fqapi/audio_proxy.go +++ b/internal/fqapi/audio_proxy.go @@ -20,6 +20,16 @@ func GenerateAudioProxyURL(originalURL, deviceID string, headers map[string]stri // 生成缓存key key := generateRandomKey(16) + // Redis 模式：TTL 自动过期 + if redisCache != nil { + saveAudioProxyEntry(key, &audioProxyEntry{ + OriginalURL: originalURL, + DeviceID: deviceID, + Headers: headers, + }) + return fmt.Sprintf("/audio/proxy/%s", key) + } + // 将headers序列化为JSON字符串 headersJSON, _ := json.Marshal(headers) @@ -28,10 +38,10 @@ func GenerateAudioProxyURL(originalURL, deviceID string, headers map[string]stri cacheMutex.Lock() defer cacheMutex.Unlock() - _, err := cacheDB.Exec(` + _, err := cacheDB.Exec(q(` INSERT INTO audio_cache (key, original_url, device_id, headers, created_at) VALUES (?, ?, ?, ?, ?) - `, key, originalURL, deviceID, string(headersJSON), time.Now().Unix()) + `), key, originalURL, deviceID, string(headersJSON), time.Now().Unix()) if err != nil { audioLogger.Error("保存音频缓存失败: " + err.Error()) } @@ -43,40 +53,50 @@ func GenerateAudioProxyURL(originalURL, deviceID string, headers map[string]stri // HandleAudioProxy 处理音频代理请求 func HandleAudioProxy(key string) ([]byte, string, error) { - if cacheDB == nil { - return nil, "", fmt.Errorf("数据库未初始化") - } - - // 从数据库获取音频信息 - var originalURL, deviceID, headersJSON string - var createdAt int64 - - cacheMutex.Lock() - err := cacheDB.QueryRow(` - SELECT original_url, device_id, headers, created_at - FROM audio_cache WHERE key = ? - `, key).Scan(&originalURL, &deviceID, &headersJSON, &createdAt) - cacheMutex.Unlock() - - if err == sql.ErrNoRows { - return nil, "", fmt.Errorf("音频信息不存在，请先调用 /audio 接口") - } - if err != nil { - return nil, "", fmt.Errorf("查询音频缓存失败: %w", err) - } - - // 检查缓存是否过期（2小时，匹配CDN链接有效期） - if time.Now().Unix()-createdAt > 7200 { // 2小时 = 7200秒 - // 清理过期缓存 - cacheMutex.Lock() - cacheDB.Exec("DELETE FROM audio_cache WHERE key = ?", key) - cacheMutex.Unlock() - return nil, "", fmt.Errorf("音频信息已过期，请重新调用 /audio 接口") - } - - // 解析headers + var originalURL, deviceID string var headers map[string]string - json.Unmarshal([]byte(headersJSON), &headers) + + if redisCache != nil { + entry, err := loadAudioProxyEntry(key) + if err != nil { + return nil, "", err + } + originalURL, deviceID, headers = entry.OriginalURL, entry.DeviceID, entry.Headers + } else { + if cacheDB == nil { + return nil, "", fmt.Errorf("数据库未初始化") + } + + // 从数据库获取音频信息 + var headersJSON string + var createdAt int64 + + cacheMutex.Lock() + err := cacheDB.QueryRow(q(` + SELECT original_url, device_id, headers, created_at + FROM audio_cache WHERE key = ? + `), key).Scan(&originalURL, &deviceID, &headersJSON, &createdAt) + cacheMutex.Unlock() + + if err == sql.ErrNoRows { + return nil, "", fmt.Errorf("音频信息不存在，请先调用 /audio 接口") + } + if err != nil { + return nil, "", fmt.Errorf("查询音频缓存失败: %w", err) + } + + // 检查缓存是否过期（2小时，匹配CDN链接有效期） + if time.Now().Unix()-createdAt > 7200 { // 2小时 = 7200秒 + // 清理过期缓存 + cacheMutex.Lock() + cacheDB.Exec(q("DELETE FROM audio_cache WHERE key = ?"), key) + cacheMutex.Unlock() + return nil, "", fmt.Errorf("音频信息已过期，请重新调用 /audio 接口") + } + + // 解析headers + json.Unmarshal([]byte(headersJSON), &headers) + } if headers == nil { headers = make(map[string]string) } @@ -92,6 +112,11 @@ func HandleAudioProxy(key string) ([]byte, string, error) { // CleanExpiredAudioCache 清理过期的音频缓存 func CleanExpiredAudioCache() { + // Redis 模式下由 TTL 自动过期，无需清理 + if redisCache != nil { + return + } + if cacheDB == nil { return } @@ -100,7 +125,7 @@ func CleanExpiredAudioCache() { defer cacheMutex.Unlock() // 删除2小时前的缓存 - result, err := cacheDB.Exec("DELETE FROM audio_cache WHERE created_at < ?", time.Now().Unix()-7200) + result, err := cacheDB.Exec(q("DELETE FROM audio_cache WHERE created_at < ?"), time.Now().Unix()-7200) if err != nil { audioLogger.Error("清理过期音频缓存失败: " + err.Error()) return diff --git a/internal/fqapi/cache_manager.go b/internal/fqapi/cache_manager.go index 765a55e..fb66ed3 100644 --- a/internal/fqapi/cache_manager.go +++ b/internal/fqapi/cache_manager.go @@ -1,19 +1,27 @@ package fqapi import ( + "context" "database/sql" "encoding/json" "fanqie/internal/common" "fanqie/internal/config" "fmt" + "net/url" + "strconv" + "strings" "sync" "time" + _ "github.com/jackc/pgx/v5/stdlib" _ "github.com/mattn/go-sqlite3" + "github.com/redis/go-redis/v9" ) var ( cacheDB *sql.DB + dbDriver string // "sqlite3" | "pgx" + redisCache *redis.Client cacheMutex sync.Mutex cacheLogger = common.NewLogger("cache_manager", "INFO", "") contentRequestCount int // content接口访问计数 @@ -21,109 +29,222 @@ var ( cacheCleanupTicker *time.Ticker // 缓存清理定时器 ) +// q 按数据库方言转换占位符（PostgreSQL 使用 $n 形式） +func q(query string) string { + if dbDriver != "pgx" { + return query + } + var b strings.Builder + n := 0 + for i := 0; i < len(query); i++ { + if query[i] == '?' { + n++ + b.WriteString("$" + strconv.Itoa(n)) + } else { + b.WriteByte(query[i]) + } + } + return b.String() +} + +// InitRedis 初始化 Redis 缓存（启用后正文缓存与音频代理缓存走 Redis） +func InitRedis() error { + cfg := config.GetConfig() + if cfg == nil || !cfg.Cache.Redis.Enabled { + return nil + } + + redisCache = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", cfg.Cache.Redis.Host, cfg.Cache.Redis.Port), + Password: cfg.Cache.Redis.Password, + DB: cfg.Cache.Redis.DB, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := redisCache.Ping(ctx).Err(); err != nil { + redisCache = nil + return fmt.Errorf("连接 Redis 失败: %w", err) + } + + cacheLogger.Info(fmt.Sprintf("Redis 缓存已启用: %s:%d db=%d", cfg.Cache.Redis.Host, cfg.Cache.Redis.Port, cfg.Cache.Redis.DB)) + return nil +} + +// CloseRedis 关闭 Redis 连接 +func CloseRedis() { + if redisCache != nil { + redisCache.Close() + } +} + // CacheData 缓存数据结构 type CacheData struct { Data string `json:"data"` Expi float64 `json:"expi"` } -// InitDB 初始化数据库 +// InitDB 初始化数据库（按 cache.backend 选择 sqlite 或 postgres） func InitDB() error { if cacheDB != nil { return nil // 已初始化 } - // 使用配置的路径 - dbPath := "./cache.db" cfg := config.GetConfig() - if cfg != nil && cfg.Cache.SQLitePath != "" { - dbPath = cfg.Cache.SQLitePath + backend := "sqlite" + dsn := "./cache.db" + if cfg != nil { + if cfg.Cache.Backend != "" { + backend = cfg.Cache.Backend + } + if backend == "postgres" { + pg := cfg.Cache.Postgres + if pg.Host == "" || pg.Database == "" { + return fmt.Errorf("cache.backend 为 postgres 但未配置 host/database") + } + // 用 net/url 构造连接串，密码中的特殊字符会被自动转义 + u := &url.URL{ + Scheme: "postgres", + User: url.UserPassword(pg.User, pg.Password), + Host: fmt.Sprintf("%s:%d", pg.Host, pg.Port), + Path: pg.Database, + RawQuery: "sslmode=" + pg.SSLMode, + } + dsn = u.String() + } else if cfg.Cache.SQLitePath != "" { + dsn = cfg.Cache.SQLitePath + } + } + + switch backend { + case "postgres": + dbDriver = "pgx" + default: + dbDriver = "sqlite3" + // ponytail: 单文件场景，WAL + 忙等待即可，无需更多调优 + dsn = dsn + "?_journal_mode=WAL&_busy_timeout=5000" } var err error - cacheDB, err = sql.Open("sqlite3", dbPath) + cacheDB, err = sql.Open(dbDriver, dsn) if err != nil { cacheLogger.Error("打开数据库失败: " + err.Error()) return err } - - // 创建缓存表（添加created_at字段用于清理） - _, err = cacheDB.Exec(`CREATE TABLE IF NOT EXISTS cache ( - key TEXT PRIMARY KEY, - value TEXT, - created_at INTEGER DEFAULT (strftime('%s', 'now')) - )`) - if err != nil { - cacheLogger.Error("创建缓存表失败: " + err.Error()) + if err := cacheDB.Ping(); err != nil { + cacheLogger.Error("连接数据库失败: " + err.Error()) return err } + cacheLogger.Info(fmt.Sprintf("数据库已连接: %s (driver=%s)", backend, dbDriver)) - - // 创建statistics表（保存请求统计，始终使用） - _, err = cacheDB.Exec(`CREATE TABLE IF NOT EXISTS statistics ( - id INTEGER PRIMARY KEY CHECK (id = 1), - total_requests INTEGER DEFAULT 0, - search_requests INTEGER DEFAULT 0, - detail_requests INTEGER DEFAULT 0, - catalog_requests INTEGER DEFAULT 0, - content_requests INTEGER DEFAULT 0, - manga_requests INTEGER DEFAULT 0, - audio_requests INTEGER DEFAULT 0, - video_requests INTEGER DEFAULT 0, - category_requests INTEGER DEFAULT 0, - updated_at INTEGER DEFAULT (strftime('%s', 'now')) - )`) - if err != nil { - cacheLogger.Error("创建statistics表失败: " + err.Error()) + if err := createTables(); err != nil { return err } + return nil +} + +// createTables 创建数据表（两套 DDL 各自兼容对应方言） +func createTables() error { + // 通用部分：statistics 初始化、索引、audio_cache 建表在两方言间一致或按 q() 转换 + tables := []struct{ name, sqliteDDL, pgDDL string }{ + {"cache", + `CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + value TEXT, + created_at INTEGER DEFAULT (strftime('%s', 'now')) + )`, + `CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + value TEXT, + created_at INTEGER DEFAULT 0 + )`}, + {"statistics", + `CREATE TABLE IF NOT EXISTS statistics ( + id INTEGER PRIMARY KEY CHECK (id = 1), + total_requests INTEGER DEFAULT 0, + search_requests INTEGER DEFAULT 0, + detail_requests INTEGER DEFAULT 0, + catalog_requests INTEGER DEFAULT 0, + content_requests INTEGER DEFAULT 0, + manga_requests INTEGER DEFAULT 0, + audio_requests INTEGER DEFAULT 0, + video_requests INTEGER DEFAULT 0, + category_requests INTEGER DEFAULT 0, + updated_at INTEGER DEFAULT (strftime('%s', 'now')) + )`, + `CREATE TABLE IF NOT EXISTS statistics ( + id INTEGER PRIMARY KEY CHECK (id = 1), + total_requests INTEGER DEFAULT 0, + search_requests INTEGER DEFAULT 0, + detail_requests INTEGER DEFAULT 0, + catalog_requests INTEGER DEFAULT 0, + content_requests INTEGER DEFAULT 0, + manga_requests INTEGER DEFAULT 0, + audio_requests INTEGER DEFAULT 0, + video_requests INTEGER DEFAULT 0, + category_requests INTEGER DEFAULT 0, + updated_at INTEGER DEFAULT 0 + )`}, + {"device_pool", + `CREATE TABLE IF NOT EXISTS device_pool ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + device_header TEXT NOT NULL, + key TEXT NOT NULL, + key_ver INTEGER NOT NULL, + is_valid INTEGER DEFAULT 1, + created_at INTEGER NOT NULL, + last_used_at INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS device_pool ( + id BIGSERIAL PRIMARY KEY, + device_header TEXT NOT NULL, + key TEXT NOT NULL, + key_ver INTEGER NOT NULL, + is_valid INTEGER DEFAULT 1, + created_at INTEGER NOT NULL, + last_used_at INTEGER NOT NULL + )`}, + {"audio_cache", + `CREATE TABLE IF NOT EXISTS audio_cache ( + key TEXT PRIMARY KEY, + original_url TEXT NOT NULL, + device_id TEXT NOT NULL, + headers TEXT, + created_at INTEGER DEFAULT (strftime('%s', 'now')) + )`, + `CREATE TABLE IF NOT EXISTS audio_cache ( + key TEXT PRIMARY KEY, + original_url TEXT NOT NULL, + device_id TEXT NOT NULL, + headers TEXT, + created_at INTEGER DEFAULT 0 + )`}, + } + + for _, t := range tables { + ddl := t.sqliteDDL + if dbDriver == "pgx" { + ddl = t.pgDDL + } + if _, err := cacheDB.Exec(ddl); err != nil { + cacheLogger.Error("创建" + t.name + "表失败: " + err.Error()) + return err + } + } // 初始化statistics表（如果不存在记录） - _, err = cacheDB.Exec(`INSERT OR IGNORE INTO statistics (id) VALUES (1)`) - if err != nil { + if _, err := cacheDB.Exec(`INSERT INTO statistics (id) VALUES (1) ON CONFLICT (id) DO NOTHING`); err != nil { cacheLogger.Error("初始化statistics表失败: " + err.Error()) } // 创建索引以提高清理效率 - _, err = cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_created_at ON cache(created_at)`) - if err != nil { + if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_created_at ON cache(created_at)`); err != nil { cacheLogger.Error("创建索引失败: " + err.Error()) } - - // 创建device_pool表（设备密钥池） - _, err = cacheDB.Exec(`CREATE TABLE IF NOT EXISTS device_pool ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_header TEXT NOT NULL, - key TEXT NOT NULL, - key_ver INTEGER NOT NULL, - is_valid INTEGER DEFAULT 1, - created_at INTEGER NOT NULL, - last_used_at INTEGER NOT NULL - )`) - if err != nil { - cacheLogger.Error("创建device_pool表失败: " + err.Error()) - return err - } - - // 创建device_pool索引 - _, err = cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`) - if err != nil { + if _, err := cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`); err != nil { cacheLogger.Warning("创建device_pool索引失败: " + err.Error()) } - // 创建audio_cache表（音频代理缓存） - _, err = cacheDB.Exec(`CREATE TABLE IF NOT EXISTS audio_cache ( - key TEXT PRIMARY KEY, - original_url TEXT NOT NULL, - device_id TEXT NOT NULL, - headers TEXT, - created_at INTEGER DEFAULT (strftime('%s', 'now')) - )`) - if err != nil { - cacheLogger.Error("创建audio_cache表失败: " + err.Error()) - return err - } - return nil } @@ -141,7 +262,11 @@ func GetCache(key string) string { if !isCacheEnabled() { return "" } - + + if redisCache != nil { + return redisGetCache(key) + } + if cacheDB == nil { return "" } @@ -150,7 +275,7 @@ func GetCache(key string) string { defer cacheMutex.Unlock() var value string - err := cacheDB.QueryRow("SELECT value FROM cache WHERE key = ?", key).Scan(&value) + err := cacheDB.QueryRow(q("SELECT value FROM cache WHERE key = ?"), key).Scan(&value) if err != nil { if err != sql.ErrNoRows { cacheLogger.Error("查询缓存失败: " + err.Error()) @@ -177,13 +302,16 @@ func GetCache(key string) string { return "" } -// UpdateCache 更新缓存 // UpdateCache 更新或插入正文内容缓存 func UpdateCache(key string, value string, expi int) error { if !isCacheEnabled() { return nil } - + + if redisCache != nil { + return redisUpdateCache(key, value, expi) + } + if cacheDB == nil { return nil } @@ -201,18 +329,9 @@ func UpdateCache(key string, value string, expi int) error { return err } - // 检查是否存在 - var exists bool - err = cacheDB.QueryRow("SELECT 1 FROM cache WHERE key = ?", key).Scan(&exists) - - if err == sql.ErrNoRows { - // 不存在，插入 - _, err = cacheDB.Exec("INSERT INTO cache (key, value) VALUES (?, ?)", key, string(jsonData)) - } else { - // 存在，更新 - _, err = cacheDB.Exec("UPDATE cache SET value = ? WHERE key = ?", string(jsonData), key) - } - + // UPSERT：ON CONFLICT 语法 SQLite 3.24+ 与 PostgreSQL 通用 + _, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`), key, string(jsonData), time.Now().Unix()) if err != nil { cacheLogger.Error("更新缓存失败: " + err.Error()) return err @@ -248,7 +367,11 @@ func CleanupAllCache() error { return err } - // 智能压缩：仅在空闲空间超过阈值时执行 + // 智能压缩：仅在空闲空间超过阈值时执行（仅 SQLite，PG 由 autovacuum 自行管理） + if dbDriver == "pgx" { + return nil + } + var freelistCount int64 err = cacheDB.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) if err == nil && freelistCount > 0 { @@ -304,7 +427,7 @@ func CleanupExpiredCache() error { expiredTime := time.Now().Unix() - int64(cfg.Cache.CleanupSeconds) // 删除过期的缓存（created_at < expiredTime） - result, err := cacheDB.Exec("DELETE FROM cache WHERE created_at < ?", expiredTime) + result, err := cacheDB.Exec(q("DELETE FROM cache WHERE created_at < ?"), expiredTime) if err != nil { cacheLogger.Error("清理过期缓存失败: " + err.Error()) return err @@ -396,6 +519,10 @@ func VacuumDatabase() error { defer cacheMutex.Unlock() cacheLogger.Info("开始执行数据库压缩...") + if dbDriver == "pgx" { + cacheLogger.Info("PostgreSQL 由 autovacuum 自动管理，无需手动压缩") + return nil + } _, err := cacheDB.Exec("VACUUM") if err != nil { cacheLogger.Error("数据库压缩失败: " + err.Error()) @@ -426,7 +553,21 @@ func GetDatabaseStats() (map[string]interface{}, error) { } stats["cache_count"] = cacheCount - // 获取数据库页面统计 + // PostgreSQL：用库大小填充，无 freelist 概念 + if dbDriver == "pgx" { + var totalSize int64 + if err := cacheDB.QueryRow("SELECT pg_database_size(current_database())").Scan(&totalSize); err == nil { + stats["page_count"] = int64(0) + stats["page_size"] = int64(0) + stats["freelist_count"] = int64(0) + stats["total_size_bytes"] = totalSize + stats["free_size_bytes"] = int64(0) + stats["used_size_bytes"] = totalSize + } + return stats, nil + } + + // SQLite：获取数据库页面统计 var pageCount, pageSize, freelistCount int64 err = cacheDB.QueryRow("PRAGMA page_count").Scan(&pageCount) if err == nil { @@ -458,8 +599,8 @@ func SaveStatistics(stats map[string]int64) error { cacheMutex.Lock() defer cacheMutex.Unlock() - _, err := cacheDB.Exec(` - UPDATE statistics + _, err := cacheDB.Exec(q(` + UPDATE statistics SET total_requests = ?, search_requests = ?, detail_requests = ?, @@ -469,10 +610,11 @@ func SaveStatistics(stats map[string]int64) error { audio_requests = ?, video_requests = ?, category_requests = ?, - updated_at = strftime('%s', 'now') + updated_at = ? WHERE id = 1 - `, stats["total"], stats["search"], stats["detail"], stats["catalog"], - stats["content"], stats["manga"], stats["audio"], stats["video"], stats["category"]) + `), stats["total"], stats["search"], stats["detail"], stats["catalog"], + stats["content"], stats["manga"], stats["audio"], stats["video"], stats["category"], + time.Now().Unix()) if err != nil { cacheLogger.Error("保存统计数据失败: " + err.Error()) diff --git a/internal/fqapi/device_key_pool.go b/internal/fqapi/device_key_pool.go index b1beb73..a96636e 100644 --- a/internal/fqapi/device_key_pool.go +++ b/internal/fqapi/device_key_pool.go @@ -385,6 +385,14 @@ func (p *DeviceKeyPool) loadFromDB() error { return nil } +// boolToInt 布尔转整数存储（is_valid 列为 INTEGER，PostgreSQL 严格要求类型匹配） +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + // saveDeviceToDB 保存设备到数据库 func (p *DeviceKeyPool) saveDeviceToDB(item *DeviceKeyItem) error { if cacheDB == nil { @@ -394,23 +402,15 @@ func (p *DeviceKeyPool) saveDeviceToDB(item *DeviceKeyItem) error { cacheMutex.Lock() defer cacheMutex.Unlock() - result, err := cacheDB.Exec(` + // RETURNING id 同时兼容 SQLite(3.35+) 与 PostgreSQL，替代 LastInsertId() + err := cacheDB.QueryRow(q(` INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at) VALUES (?, ?, ?, ?, ?, ?) - `, item.DeviceHeader, item.Key, item.KeyVer, item.IsValid, - item.CreatedAt.Unix(), item.LastUsedAt.Unix()) - - if err != nil { - return err - } + RETURNING id + `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), + item.CreatedAt.Unix(), item.LastUsedAt.Unix()).Scan(&item.ID) - // 获取插入的ID - id, err := result.LastInsertId() - if err == nil { - item.ID = int(id) - } - - return nil + return err } // updateDeviceInDB 更新设备信息 @@ -422,11 +422,11 @@ func (p *DeviceKeyPool) updateDeviceInDB(item *DeviceKeyItem) error { cacheMutex.Lock() defer cacheMutex.Unlock() - _, err := cacheDB.Exec(` - UPDATE device_pool + _, err := cacheDB.Exec(q(` + UPDATE device_pool SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ? WHERE id = ? - `, item.DeviceHeader, item.Key, item.KeyVer, item.IsValid, + `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), item.LastUsedAt.Unix(), item.ID) return err @@ -454,35 +454,6 @@ func (p *DeviceKeyPool) removeInvalidFromDB() error { return nil } -// InitDevicePoolTable 初始化设备池表 -func InitDevicePoolTable() error { - if cacheDB == nil { - return fmt.Errorf("数据库未初始化") - } - - _, err := cacheDB.Exec(`CREATE TABLE IF NOT EXISTS device_pool ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_header TEXT NOT NULL, - key TEXT NOT NULL, - key_ver INTEGER NOT NULL, - is_valid INTEGER DEFAULT 1, - created_at INTEGER NOT NULL, - last_used_at INTEGER NOT NULL - )`) - - if err != nil { - return err - } - - // 创建索引 - _, err = cacheDB.Exec(`CREATE INDEX IF NOT EXISTS idx_device_pool_valid ON device_pool(is_valid)`) - if err != nil { - poolLogger.Warning("创建索引失败: " + err.Error()) - } - - return nil -} - // GetDeviceHeaderFromPool 从池中获取设备header（兼容旧接口） func GetDeviceHeaderFromPool() string { pool := GetDeviceKeyPool() diff --git a/internal/fqapi/rebind_test.go b/internal/fqapi/rebind_test.go new file mode 100644 index 0000000..23c80d1 --- /dev/null +++ b/internal/fqapi/rebind_test.go @@ -0,0 +1,20 @@ +package fqapi + +import "testing" + +// TestRebind 验证占位符方言转换 +func TestRebind(t *testing.T) { + in := "INSERT INTO cache (key, value) VALUES (?, ?)" + + if got := q(in); got != in { + t.Errorf("sqlite 下应保持 ?: %s", got) + } + + dbDriver = "pgx" + defer func() { dbDriver = "sqlite3" }() + + want := "INSERT INTO cache (key, value) VALUES ($1, $2)" + if got := q(in); got != want { + t.Errorf("pg 下应转换占位符: got %s want %s", got, want) + } +} diff --git a/internal/fqapi/store_redis.go b/internal/fqapi/store_redis.go new file mode 100644 index 0000000..b450762 --- /dev/null +++ b/internal/fqapi/store_redis.go @@ -0,0 +1,85 @@ +package fqapi + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// audioProxyEntry 音频代理缓存条目（Redis 版） +type audioProxyEntry struct { + OriginalURL string `json:"original_url"` + DeviceID string `json:"device_id"` + Headers map[string]string `json:"headers"` +} + +const ( + redisCacheKeyPrefix = "fq:cache:" + redisAudioKeyPrefix = "fq:audio:" + // audioCacheTTL 音频代理缓存有效期（匹配 CDN 链接有效期） + audioCacheTTL = 2 * time.Hour +) + +// redisGetCache 从 Redis 获取正文缓存 +func redisGetCache(key string) string { + val, err := redisCache.Get(context.Background(), redisCacheKeyPrefix+key).Result() + if err != nil { + return "" // miss 或错误都视为未命中 + } + + var cacheData CacheData + if err := json.Unmarshal([]byte(val), &cacheData); err != nil { + cacheLogger.Error("解析Redis缓存数据失败: " + err.Error()) + return "" + } + + cacheLogger.Debug("Redis缓存命中: " + key) + if time.Now().Unix() < int64(cacheData.Expi) { + return cacheData.Data + } + return "" +} + +// redisUpdateCache 写入 Redis 正文缓存（TTL 与过期时间一致） +func redisUpdateCache(key string, value string, expi int) error { + cacheData := CacheData{ + Data: value, + Expi: float64(time.Now().Unix() + int64(expi)), + } + jsonData, err := json.Marshal(cacheData) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return redisCache.Set(ctx, redisCacheKeyPrefix+key, jsonData, time.Duration(expi)*time.Second).Err() +} + +// saveAudioProxyEntry 将音频代理信息存入 Redis +func saveAudioProxyEntry(key string, entry *audioProxyEntry) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + jsonData, _ := json.Marshal(entry) + if err := redisCache.Set(ctx, redisAudioKeyPrefix+key, jsonData, audioCacheTTL).Err(); err != nil { + audioLogger.Error("保存音频缓存到Redis失败: " + err.Error()) + } +} + +// loadAudioProxyEntry 从 Redis 读取音频代理信息，未命中或过期返回错误 +func loadAudioProxyEntry(key string) (*audioProxyEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + val, err := redisCache.Get(ctx, redisAudioKeyPrefix+key).Result() + if err != nil { + return nil, fmt.Errorf("音频信息不存在或已过期，请重新调用 /audio 接口") + } + + var entry audioProxyEntry + if err := json.Unmarshal([]byte(val), &entry); err != nil { + return nil, fmt.Errorf("解析音频缓存失败: %w", err) + } + return &entry, nil +}