diff --git a/README.md b/README.md index 427c395..627391c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,12 @@ Go 语言实现的番茄小说非官方 API 服务，完整实现了签名算法 - 分类首页数据 - 分类详情页（支持筛选、排序、分页） +### 榜单与个人数据 + +- 🏆 榜单（推荐 / 完结 / 热搜 / 黑马 / 爆更） +- 📚 我的书架（按分组浏览，需 sessionid） +- 🕐 阅读历史（含阅读进度，需 sessionid） + ### 技术特性 - **签名算法** - Gorgon、8404、Argus、Ladon、Khronos @@ -171,7 +177,23 @@ go run ./cmd/fanqie | 接口 | 方法 | 说明 | 参数 | | ------ | ------ | ------ | ------ | | `/category-front` | GET | 分类首页 | - | -| `/category-landing` | GET | 分类详情 | `category_id`, `word_number`, `book_status`, `sort_by`, `offset` | +| `/category-landing` | GET | 分类详情 | `category_id`, `word_number`, `book_status`, `sort_by`, `offset`, `selected_items`（新版分类标签自带筛选项，透传上游） | + +### 榜单与个人数据 + +| 接口 | 方法 | 说明 | 参数 | +| ------ | ------ | ------ | ------ | +| `/rank` | GET | 榜单（免签名接口） | `side_type`（`10` 推荐 / `11` 完结 / `12` 热搜 / `13` 黑马 / `15` 爆更，默认推荐）, `page`（默认 1，每页 20 本） | +| `/bookshelf` | GET | 我的书架（需登录） | `sessionid`（必填，见下方「获取 sessionid」）, `group`（可选，按分组过滤；响应含每本书的 `group_name`） | +| `/read-history` | GET | 阅读历史（需登录） | `sessionid`（必填）, `offset`（默认 0，每页 20 条；响应每项的 `history` 字段保留上游原始记录，含阅读进度） | + +#### 获取 sessionid + +番茄网页版登录由字节风控（滑块验证）保护，无法通过 API 直接登录（书源实现同样依赖浏览器人工登录）。获取方式： + +1. 浏览器打开 [fanqienovel.com](https://fanqienovel.com/) 并登录（手机验证码或密码） +2. F12 → Application → Cookies → 复制 `sessionid` 的值（有效期较长，可长期使用） +3. 直接传入 `/bookshelf?sessionid=...` 或 `/read-history?sessionid=...` 使用（书架为空多半是 sessionid 无效或已过期） ### 媒体相关 @@ -288,7 +310,7 @@ cache: password: "" # 密码 database: "fanqie" # 数据库名 sslmode: "disable" # SSL 模式: disable | prefer | require - redis: # Redis 缓存（正文缓存与音频代理缓存，统计数据/设备池仍在 SQL 库） + redis: # Redis 热缓存层（启用后与 SQL 双写：Redis 短 TTL + SQL 持久层 30 天） enabled: false host: "127.0.0.1" # 主机 port: 6379 # 端口 @@ -388,15 +410,19 @@ cp .env.example .env ## 缓存机制 -- **接口缓存** - 命中直接返回，不打上游（Redis 启用时优先走 Redis） -- **自动清理** - 定期清理过期缓存（Redis 由 TTL 自动过期） +Redis 启用时为**双层架构**：Redis 热层（TTL=业务缓存时长）+ SQL 持久层（不可变内容固定保留 30 天）。 + +- **读**：Redis → 未命中查 SQL（命中自动回填 Redis）→ 回源上游 +- **写**：Redis + SQL 双写；搜索/分类等动态短缓存只走 Redis 不落库 +- **防击穿**：singleflight 合并并发未命中请求，同一 key 只有一个请求打上游 +- **自动清理** - 定期按行内过期时间清理 SQL 过期缓存（Redis 由 TTL 自动过期） - **数据库压缩** - 支持 VACUUM 操作优化数据库 | 接口 | 缓存时长 | | ------ | ------ | -| `/content` | 24 小时 | -| `/detail`、`/info`、`/catalog`、`/multi-detail`、`/item-summary` | 24 小时 | -| `/search`、`/category-front`、`/category-landing` | 10 分钟 | +| `/content`、`/multi-content` | 24 小时（SQL 层 30 天） | +| `/detail`、`/info`、`/catalog`、`/multi-detail`、`/item-summary` | 24 小时（SQL 层 30 天） | +| `/search`、`/category-front`、`/category-landing` | 10 分钟（仅 Redis） | | `/audio` 代理链接 | 2 小时（匹配 CDN 有效期） | 失败响应不缓存；评论类接口为动态数据，不缓存。 diff --git a/cmd/fanqie/main.go b/cmd/fanqie/main.go index daf85cf..b3a42db 100644 --- a/cmd/fanqie/main.go +++ b/cmd/fanqie/main.go @@ -76,6 +76,7 @@ func main() { logger.Error("设备密钥池初始化失败: " + err.Error()) os.Exit(1) } + defer fqapi.FlushDevicePool() // 程序退出时落盘设备使用记录 // 设置路由 setupRoutes() @@ -164,6 +165,9 @@ func setupRoutes() { http.HandleFunc("/catalog", wrap(handlers.HandleCatalog)) http.HandleFunc("/directory/all_items", wrap(handlers.HandleCatalog)) http.HandleFunc("/multi-detail", wrap(handlers.HandleMultiDetail)) + http.HandleFunc("/rank", wrap(handlers.HandleRank)) + http.HandleFunc("/bookshelf", wrap(handlers.HandleBookshelf)) + http.HandleFunc("/read-history", wrap(handlers.HandleReadHistory)) http.HandleFunc("/multi-content", wrap(handlers.HandleMultiContent)) http.HandleFunc("/content", wrap(handlers.HandleContent)) http.HandleFunc("/item-summary", wrap(handlers.HandleItemSummary)) diff --git a/go.mod b/go.mod index 4eb657f..c3c485f 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.18 github.com/redis/go-redis/v9 v9.22.0 github.com/spf13/viper v1.18.2 + golang.org/x/sync v0.17.0 golang.org/x/time v0.5.0 ) @@ -35,7 +36,6 @@ require ( go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.46.0 // indirect - golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/internal/algos/signer.go b/internal/algos/signer.go index 53df54d..1778684 100644 --- a/internal/algos/signer.go +++ b/internal/algos/signer.go @@ -74,6 +74,7 @@ func Sign4( // 添加 Ladon 和 Argus 签名 ladon := &Ladon{} result["x-ladon"] = ladon.Encrypt(timestamp, licenseID, aid) + result["x-helios"] = result["x-ladon"] // X-Helios 与 X-Ladon 同值（对齐书源实现） argus := &Argus{} result["x-argus"] = argus.GetSign( diff --git a/internal/fqapi/cache_manager.go b/internal/fqapi/cache_manager.go index e0ae5ad..f350ffc 100644 --- a/internal/fqapi/cache_manager.go +++ b/internal/fqapi/cache_manager.go @@ -16,8 +16,31 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" _ "github.com/mattn/go-sqlite3" "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" ) +const ( + // sqlCacheTTLSeconds SQL 持久层保留时长：不可变内容（正文/详情/目录）与业务 TTL 解耦， + // Redis 过期或重启后由 SQL 回填，避免重新打上游 + sqlCacheTTLSeconds = 30 * 86400 +) + +// volatileCachePrefixes 动态短缓存（值会变）只走 Redis/短 TTL，不进 SQL 长期层 +var volatileCachePrefixes = []string{"search_", "category_", "rank_"} + +// isVolatileCacheKey 判断是否为动态短缓存 key +func isVolatileCacheKey(key string) bool { + for _, p := range volatileCachePrefixes { + if strings.HasPrefix(key, p) { + return true + } + } + return false +} + +// cacheSF 合并并发 miss 的回源请求，防止缓存击穿 +var cacheSF singleflight.Group + var ( cacheDB *sql.DB dbDriver string // "sqlite3" | "pgx" @@ -56,6 +79,11 @@ func InitRedis() error { Addr: fmt.Sprintf("%s:%d", cfg.Cache.Redis.Host, cfg.Cache.Redis.Port), Password: cfg.Cache.Redis.Password, DB: cfg.Cache.Redis.DB, + // 缓存场景失败不重试：miss 直接降级 SQL，重试只会放大网络慢时的请求延迟 + MaxRetries: 0, + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + MinRetryBackoff: 0, }) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -149,12 +177,14 @@ func createTables() error { `CREATE TABLE IF NOT EXISTS cache ( key TEXT PRIMARY KEY, value TEXT, + expi INTEGER DEFAULT 0, created_at INTEGER DEFAULT (strftime('%s', 'now')), updated_at INTEGER DEFAULT (strftime('%s', 'now')) )`, `CREATE TABLE IF NOT EXISTS cache ( key TEXT PRIMARY KEY, value TEXT, + expi BIGINT DEFAULT 0, created_at BIGINT DEFAULT extract(epoch FROM now())::bigint, updated_at BIGINT DEFAULT extract(epoch FROM now())::bigint )`}, @@ -259,8 +289,10 @@ func createTables() error { // 旧表补 created_at/updated_at 列（新表 DDL 自带，此处仅老库生效） for _, m := range []struct{ table, col, ddl string }{ {"cache", "updated_at", "INTEGER DEFAULT 0"}, // SQLite ALTER 不允许非常量默认值，先补 0 再回填 + {"cache", "expi", "INTEGER DEFAULT 0"}, {"statistics", "created_at", "INTEGER DEFAULT 0"}, {"device_pool", "updated_at", "INTEGER DEFAULT 0"}, + {"device_pool", "key_ts", "INTEGER DEFAULT 0"}, {"audio_cache", "updated_at", "INTEGER DEFAULT 0"}, } { if dbDriver == "pgx" { @@ -278,6 +310,7 @@ func createTables() error { // 旧行回填（幂等：仅 0 值行受影响） for _, backfill := range []string{ `UPDATE cache SET updated_at = created_at WHERE updated_at = 0`, + `UPDATE cache SET expi = created_at + 86400 WHERE expi = 0`, `UPDATE statistics SET created_at = updated_at WHERE created_at = 0`, `UPDATE device_pool SET updated_at = last_used_at WHERE updated_at = 0`, `UPDATE audio_cache SET updated_at = created_at WHERE updated_at = 0`, @@ -299,7 +332,8 @@ func isCacheEnabled() bool { return cfg.Cache.Enabled } -// withCache 只读接口的缓存包装：命中直接返回，miss 回源后写入（仅缓存成功响应） +// withCache 只读接口的缓存包装：命中直接返回，miss 回源后写入（仅缓存成功响应）。 +// singleflight 合并并发 miss，同一 key 只有一个请求打上游 // ponytail: 空参等防御分支若漏到此处，其错误 JSON 也会被缓存到 TTL 到期——handler 层 validateParams 已拦截 func withCache(key string, ttl int, fetch func() (string, error)) (string, error) { if c := GetCache(key); c != "" { @@ -307,21 +341,36 @@ func withCache(key string, ttl int, fetch func() (string, error)) (string, error // （对新写入的已解包数据无影响） return reencodeUnwrapped(c), nil } - result, err := fetch() - if err == nil { - UpdateCache(key, result, ttl) + v, err, _ := cacheSF.Do(key, func() (interface{}, error) { + // 双重检查：等锁期间可能已被并发请求填充 + if c := GetCache(key); c != "" { + return c, nil + } + result, err := fetch() + if err == nil { + UpdateCache(key, result, ttl) + } + return result, err + }) + if err != nil { + return "", err } - return result, err + return v.(string), nil } -// GetCache 获取正文内容缓存 +// GetCache 获取缓存：Redis 热层 → SQL 持久层（命中回填 Redis）→ 未命中 func GetCache(key string) string { if !isCacheEnabled() { return "" } if redisCache != nil { - return redisGetCache(key) + if v := redisGetCache(key); v != "" { + return v + } + if isVolatileCacheKey(key) { + return "" // 搜索/分类等动态数据不落库，Redis 未命中即回源 + } } if cacheDB == nil { @@ -329,7 +378,8 @@ func GetCache(key string) string { } var value string - err := cacheDB.QueryRow(q("SELECT value FROM cache WHERE key = ?"), key).Scan(&value) + var expi int64 + err := cacheDB.QueryRow(q("SELECT value, expi FROM cache WHERE key = ?"), key).Scan(&value, &expi) if err != nil { if err != sql.ErrNoRows { cacheLogger.Error("查询缓存失败: " + err.Error()) @@ -337,33 +387,46 @@ func GetCache(key string) string { return "" } + // 检查是否过期（expi 为 0 的旧行由启动回填兜底，此处防御） + if time.Now().Unix() >= expi { + cacheLogger.Debug("缓存已过期(SQL): " + key) + return "" + } + var cacheData CacheData - err = json.Unmarshal([]byte(value), &cacheData) - if err != nil { + if err := json.Unmarshal([]byte(value), &cacheData); err != nil { cacheLogger.Error("解析缓存数据失败: " + err.Error()) return "" } - cacheLogger.Debug("缓存命中: " + key) + cacheLogger.Debug("缓存命中(SQL): " + key) - // 检查是否过期 - if time.Now().Unix() < int64(cacheData.Expi) { - cacheLogger.Debug("缓存有效！") - return cacheData.Data + // SQL 命中而 Redis 未命中（过期/重启）：异步回填热层，剩余时长为 TTL， + // 回填（尤其大 value）不阻塞本次请求 + if redisCache != nil { + remain := int(expi - time.Now().Unix()) + if remain < 60 { + remain = 60 + } + go redisUpdateCache(key, cacheData.Data, remain) } - - cacheLogger.Debug("缓存已过期！") - return "" + return cacheData.Data } -// UpdateCache 更新或插入正文内容缓存 +// UpdateCache 写入缓存：Redis 热层（TTL=业务过期时间）+ SQL 持久层双写。 +// 不可变内容在 SQL 固定保留 30 天，动态短缓存（search/category）只走 Redis。 +// Redis 写失败只降级不阻断：持久层照常落库，热层下次请求回填 func UpdateCache(key string, value string, expi int) error { if !isCacheEnabled() { return nil } if redisCache != nil { - return redisUpdateCache(key, value, expi) + if err := redisUpdateCache(key, value, expi); err != nil { + cacheLogger.Warning(fmt.Sprintf("Redis 写入失败（降级为仅 SQL 落库）: %v", err)) + } else if isVolatileCacheKey(key) { + return nil // 动态短缓存不落库（Redis 成功即止） + } } if cacheDB == nil { @@ -380,10 +443,17 @@ func UpdateCache(key string, value string, expi int) error { return err } + // SQL 保留时长：不可变内容固定 30 天（Redis 未启用时沿用业务 TTL，保持纯 SQL 模式原行为） + sqlTTL := int64(sqlCacheTTLSeconds) + if redisCache == nil || isVolatileCacheKey(key) { + sqlTTL = int64(expi) + } + // UPSERT：ON CONFLICT 语法 SQLite 3.24+ 与 PostgreSQL 通用 - _, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, created_at, updated_at) VALUES (?, ?, ?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`), - key, string(jsonData), time.Now().Unix(), time.Now().Unix()) + now := time.Now().Unix() + _, err = cacheDB.Exec(q(`INSERT INTO cache (key, value, expi, created_at, updated_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value, expi = excluded.expi, updated_at = excluded.updated_at`), + key, string(jsonData), now+sqlTTL, now, now) if err != nil { cacheLogger.Error("更新缓存失败: " + err.Error()) return err @@ -465,11 +535,8 @@ func CleanupExpiredCache() error { return fmt.Errorf("数据库未初始化") } - // 计算过期时间点（当前时间 - cleanup_seconds） - expiredTime := time.Now().Unix() - int64(cfg.Cache.CleanupSeconds) - - // 删除过期的缓存（created_at < expiredTime） - result, err := cacheDB.Exec(q("DELETE FROM cache WHERE created_at < ?"), expiredTime) + // 删除已过期的缓存（按行内过期时间 expi，而非创建时间） + result, err := cacheDB.Exec(q("DELETE FROM cache WHERE expi < ?"), time.Now().Unix()) if err != nil { cacheLogger.Error("清理过期缓存失败: " + err.Error()) return err @@ -480,7 +547,7 @@ func CleanupExpiredCache() error { if err != nil { cacheLogger.Warning("获取删除行数失败: " + err.Error()) } else if rowsAffected > 0 { - cacheLogger.Info(fmt.Sprintf("已清理 %d 条过期缓存（创建时间早于 %d 秒前）", rowsAffected, cfg.Cache.CleanupSeconds)) + cacheLogger.Info(fmt.Sprintf("已清理 %d 条过期缓存（SQL 持久层）", rowsAffected)) } // 顺带清理磁盘视频缓存（7 天未访问的解密文件） diff --git a/internal/fqapi/cache_manager_test.go b/internal/fqapi/cache_manager_test.go index 691f97d..06f5ab3 100644 --- a/internal/fqapi/cache_manager_test.go +++ b/internal/fqapi/cache_manager_test.go @@ -30,6 +30,7 @@ func TestConcurrentCacheAccess(t *testing.T) { if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache ( key TEXT PRIMARY KEY, value TEXT, + expi INTEGER DEFAULT 0, created_at INTEGER DEFAULT (strftime('%s', 'now')), updated_at INTEGER DEFAULT (strftime('%s', 'now')) )`); err != nil { @@ -74,6 +75,7 @@ func TestWithCache(t *testing.T) { }() if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS cache ( key TEXT PRIMARY KEY, value TEXT, + expi INTEGER DEFAULT 0, created_at INTEGER DEFAULT (strftime('%s', 'now')), updated_at INTEGER DEFAULT (strftime('%s', 'now')) )`); err != nil { diff --git a/internal/fqapi/category_front.go b/internal/fqapi/category_front.go index 9903853..f04b532 100644 --- a/internal/fqapi/category_front.go +++ b/internal/fqapi/category_front.go @@ -17,10 +17,13 @@ func HandleCategoryFront(tab interface{}) (string, error) { func fetchCategoryFront(tab interface{}) (string, error) { tabInt := ParseIntParam(tab, 0) + // 新版分类接口参数（category_new_page_715=1 + new_category_type， + // 旧参数 new_category_tab 已失效——对齐 2026-10 可用的书源实现） reqURL := BuildURL( "https://reading.snssdk.com/reading/bookapi/new_category/front/v/", - fmt.Sprintf("new_category_tab=%d", tabInt), - fmt.Sprintf("update_version_code=%s", "58932"), + "category_new_page_715=1", + fmt.Sprintf("new_category_type=%d", tabInt), + "update_version_code=" + ClientVersionCode, "distinct_style=1", ) diff --git a/internal/fqapi/category_landing.go b/internal/fqapi/category_landing.go index 959cf47..7023796 100644 --- a/internal/fqapi/category_landing.go +++ b/internal/fqapi/category_landing.go @@ -8,32 +8,54 @@ import ( var categoryLandingLogger = common.NewLogger("category_landing", "INFO", "") // HandleCategoryLanding 处理获取分类详情,分类详情短缓存 10 分钟 -func HandleCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy interface{}) (string, error) { - return withCache(fmt.Sprintf("category_landing_%v_%v_%v_%v_%v", categoryID, offset, wordNumber, bookStatus, sortBy), 600, func() (string, error) { - return fetchCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy) +// selectedItems：新版分类标签自带的筛选项（selected_items，透传上游） +func HandleCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy, selectedItems interface{}) (string, error) { + return withCache(fmt.Sprintf("category_landing_%v_%v_%v_%v_%v_%v", categoryID, offset, wordNumber, bookStatus, sortBy, selectedItems), 600, func() (string, error) { + return fetchCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy, selectedItems) }) } -func fetchCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy interface{}) (string, error) { +func fetchCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy, selectedItems interface{}) (string, error) { // 参数转换 categoryIDInt := ParseIntParam(categoryID, 0) offsetInt := ParseIntParam(offset, 0) wordNumberInt := ParseIntParam(wordNumber, 0) bookStatusInt := ParseIntParam(bookStatus, 0) sortByInt := ParseIntParam(sortBy, 0) + selectedItemsStr := ParseStringParam(selectedItems, "") - if categoryIDInt == 0 { + if categoryIDInt == 0 && selectedItemsStr == "" { return BuildErrorResponseWithCode(100144, "参数错误"), nil } + params := []string{ + fmt.Sprintf("limit=%d", 20), + fmt.Sprintf("offset=%d", offsetInt), + } + if categoryIDInt != 0 { + params = append(params, fmt.Sprintf("category_id=%d", categoryIDInt)) + } + if wordNumberInt != 0 { + params = append(params, fmt.Sprintf("word_number=%d", wordNumberInt)) + } + if bookStatusInt != 0 { + params = append(params, fmt.Sprintf("book_status=%d", bookStatusInt)) + } + if sortByInt != 0 { + params = append(params, fmt.Sprintf("sort_by=%d", sortByInt)) + } + // 新版分类标签：selected_items 由 front 页标签自带，需与其 from_front_page 配对透传 + if selectedItemsStr != "" { + params = append(params, + "selected_items="+selectedItemsStr, + "selected_item_from_front_page="+selectedItemsStr, + "source=front_category", + ) + } + reqURL := BuildURL( "https://reading.snssdk.com/reading/bookapi/new_category/landing/v/", - fmt.Sprintf("category_id=%d", categoryIDInt), - fmt.Sprintf("limit=%d", 20), - fmt.Sprintf("offset=%d", offsetInt), - fmt.Sprintf("word_number=%d", wordNumberInt), - fmt.Sprintf("book_status=%d", bookStatusInt), - fmt.Sprintf("sort_by=%d", sortByInt), + params..., ) resp, err := GorgonRequest(reqURL, map[string]interface{}{ diff --git a/internal/fqapi/content.go b/internal/fqapi/content.go index 6e50ede..1d00ef2 100644 --- a/internal/fqapi/content.go +++ b/internal/fqapi/content.go @@ -5,12 +5,14 @@ import ( "encoding/json" "fanqie/internal/common" "fmt" - "sort" "strings" + + "golang.org/x/sync/singleflight" ) var ( contentLogger = common.NewLogger("content", "INFO", "") + contentSF singleflight.Group // 合并同一章节的并发回源请求，防缓存击穿 ) // ContentFetcher 内容获取器 @@ -26,19 +28,27 @@ func HandleGetContent(itemID string, textMode, imageMode interface{}) (string, e textModeBool := parseBoolParamDefault(textMode, getContentConfig().RemoveHTMLTags) imageModeBool := parseBoolParamDefault(imageMode, !getContentConfig().RemoveImages) - // contentLogger.Info(fmt.Sprintf("获取内容 item_id=%s text_mode=%v image_mode=%v", itemID, textModeBool, imageModeBool)) - - // 检查缓存 cache := GetCache("content_" + itemID) - if cache != "" { - return processCachedContent(cache, textModeBool, imageModeBool) + if cache == "" { + // 同一章节并发请求只打一次上游，其余等待共享结果 + v, err, _ := contentSF.Do("content_"+itemID, func() (interface{}, error) { + // 双重检查：等锁期间可能已被并发请求填充 + if c := GetCache("content_" + itemID); c != "" { + return c, nil + } + return fetchContentFromAPI(itemID) + }) + if err != nil { + return "", err + } + cache = v.(string) } - // 缓存未命中，从API获取 - return fetchContentFromAPI(itemID, textModeBool, imageModeBool) + // 出口按各自参数解析（缓存/回源共用同一路径） + return processCachedContent(cache, textModeBool, imageModeBool) } -// processCachedContent 处理缓存内容 +// processCachedContent 处理缓存内容（content 为解密后的原始正文） func processCachedContent(cache string, textMode, imageMode bool) (string, error) { var d map[string]interface{} json.Unmarshal([]byte(cache), &d) @@ -53,18 +63,18 @@ func processCachedContent(cache string, textMode, imageMode bool) (string, error return buildJSONResponse(UnwrapJSONStrings(d)), nil } -// fetchContentFromAPI 从API获取内容（带设备切换重试） -func fetchContentFromAPI(itemID string, textMode, imageMode bool) (string, error) { - return fetchContentWithRetry(itemID, textMode, imageMode, 0) +// fetchContentFromAPI 从API获取内容（带设备切换重试），返回缓存格式 JSON +func fetchContentFromAPI(itemID string) (string, error) { + return fetchContentWithRetry(itemID, 0) } // fetchContentWithRetry 从API获取内容，支持设备切换重试 -func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount int) (string, error) { +func fetchContentWithRetry(itemID string, retryCount int) (string, error) { // 防止无限递归 const maxRetries = 5 if retryCount >= maxRetries { contentLogger.Warning(fmt.Sprintf("已重试 %d 次，尝试备用接口", maxRetries)) - return tryBackupAPI(itemID, textMode, imageMode) + return tryBackupAPI(itemID) } // 从设备密钥池获取设备 @@ -79,6 +89,7 @@ func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount i currentKeyInfo := &KeyInfo{ Key: item.Key, Ver: item.KeyVer, + Ts: item.KeyTs, } contentLogger.Debug(fmt.Sprintf("使用池中设备 ID=%d (重试次数: %d)", item.ID, retryCount)) @@ -90,16 +101,13 @@ func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount i "app_name=novelapp", "channel=0", "device_platform=android", - "version_code=66.9", + "version_code=" + ClientVersionCode, "item_id=" + itemID, "os_version=0", + fmt.Sprintf("key_register_ts=%d", currentKeyInfo.Ts), // 降低上游密钥不匹配概率 headerWithoutVersion, } - - allParams := strings.Join(params, "&") - paramList := strings.Split(allParams, "&") - sort.Strings(paramList) - sortedParams := strings.Join(paramList, "&") + sortedParams := buildSortedQuery(params...) url := "https://reading.snssdk.com/reading/reader/full/v/?" + sortedParams @@ -114,7 +122,7 @@ func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount i contentLogger.Warning(fmt.Sprintf("主接口请求出错，标记设备无效并切换设备: %v", err)) pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } var body map[string]interface{} @@ -126,7 +134,7 @@ func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount i contentLogger.Warning("响应data字段无效，切换设备重试") pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } contentEncoded, ok := data["content"].(string) @@ -135,76 +143,101 @@ func fetchContentWithRetry(itemID string, textMode, imageMode bool, retryCount i contentLogger.Warning(fmt.Sprintf("内容返回 Invalid，标记设备无效并切换设备 (重试: %d)", retryCount)) pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } // 检查密钥版本 keyVersion, _ := data["key_version"].(float64) - if currentKeyInfo != nil && int(keyVersion) != currentKeyInfo.Ver { + if int(keyVersion) != currentKeyInfo.Ver { contentLogger.Warning("密钥版本不匹配，重新获取密钥") // 更新池中的密钥 newKeyInfo, err := GetKey(header) if err == nil { - UpdateKeyInPool(header, newKeyInfo.Key, newKeyInfo.Ver) + UpdateKeyInPool(header, newKeyInfo) currentKeyInfo = newKeyInfo } else { // 获取密钥失败，切换设备 contentLogger.Warning("获取密钥失败，切换设备重试") pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } } - // 解密内容 - contentBytes, _ := base64.StdEncoding.DecodeString(contentEncoded) - if len(contentBytes) < 16 { - contentLogger.Warning("内容长度不足，切换设备重试") - pool.MarkInvalid(header) - pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) - } - - iv := contentBytes[0:16] - encryptedData := contentBytes[16:] - - keyBytes, _ := base64.StdEncoding.DecodeString(currentKeyInfo.Key) - decrypted, err := De(encryptedData, keyBytes, iv) + contentStr, err := decodeContentBody(data, currentKeyInfo) if err != nil { contentLogger.Warning(fmt.Sprintf("解密失败，切换设备重试: %v", err)) pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } - // 处理内容 - contentStr := string(decrypted) - contentStr = strings.Replace(contentStr, ``, "", -1) + contentStr = strings.ReplaceAll(contentStr, ``, "") contentStr = strings.TrimSpace(contentStr) data["content"] = contentStr - // 更新缓存 - cacheData := buildCacheData(data) - saveCacheJSON("content_"+itemID, cacheData, 86400*2) - - // 解析内容 - parser := NewContentParser() - data["content"] = parser.Parse(contentStr, textMode, imageMode) - - return buildJSONResponse(UnwrapJSONStrings(body)), nil + // 写缓存并返回缓存格式（按参数的解析统一放在出口 processCachedContent） + cacheStr := marshalNoEscape(buildCacheData(UnwrapJSONStrings(data))) + UpdateCache("content_"+itemID, cacheStr, 86400) + return cacheStr, nil } // 响应code不为0，切换设备重试 contentLogger.Warning(fmt.Sprintf("响应code异常: %.0f，切换设备重试", body["code"])) pool.MarkInvalid(header) pool.RemoveInvalid() - return fetchContentWithRetry(itemID, textMode, imageMode, retryCount+1) + return fetchContentWithRetry(itemID, retryCount+1) } -// tryBackupAPI 尝试备用API（所有设备都失败后的最后手段） -func tryBackupAPI(itemID string, textMode, imageMode bool) (string, error) { +// decodeContentBody 按上游 crypt_status 解出正文： +// - 0：base64 密文，设备密钥 AES-CBC 解密（compress_status!=0 时解压） +// - 1：明文（可能压缩） +// - 2：密钥已失效，刷新密钥后由调用方重试 +func decodeContentBody(data map[string]interface{}, keyInfo *KeyInfo) (string, error) { + contentEncoded, _ := data["content"].(string) + cryptStatus, _ := data["crypt_status"].(float64) + compressStatus, _ := data["compress_status"].(float64) + + switch int(cryptStatus) { + case 1: + if compressStatus != 0 { + return string(gunzipMaybe([]byte(contentEncoded))), nil + } + return contentEncoded, nil + + case 2: + return "", fmt.Errorf("密钥已失效（crypt_status=2），需刷新密钥重试") + + default: + // 500003 场景（设备无需解密）仍下发密文时按明文兜底 + if keyInfo == nil || keyInfo.Key == "" { + return contentEncoded, nil + } + + contentBytes, err := base64.StdEncoding.DecodeString(contentEncoded) + if err != nil || len(contentBytes) < 16 { + return "", fmt.Errorf("内容密文长度异常") + } + + iv := contentBytes[0:16] + encryptedData := contentBytes[16:] + + keyBytes, _ := base64.StdEncoding.DecodeString(keyInfo.Key) + decrypted, err := De(encryptedData, keyBytes, iv) + if err != nil { + return "", err + } + if compressStatus != 0 { + decrypted = gunzipMaybe(decrypted) + } + return string(decrypted), nil + } +} + +// tryBackupAPI 尝试备用API（所有设备都失败后的最后手段），返回缓存格式 JSON +func tryBackupAPI(itemID string) (string, error) { contentLogger.Warning("尝试备用接口1（所有设备均失败）") // 从设备池获取任意一个有效设备用于备用接口 @@ -217,8 +250,8 @@ func tryBackupAPI(itemID string, textMode, imageMode bool) (string, error) { header := item.DeviceHeader // 备用接口1：使用DH加密 - backupAPI := fmt.Sprintf("https://novel.snssdk.com/api/novel/book/reader/content/v1?support_image=1&item_id=%s&aid=1319&app_name=novelapp&channel=0&device_platform=android&os_version=2&version_code=58932&%s", - itemID, header) + backupAPI := fmt.Sprintf("https://novel.snssdk.com/api/novel/book/reader/content/v1?support_image=1&item_id=%s&aid=1319&app_name=novelapp&channel=0&device_platform=android&os_version=2&version_code=%s&%s", + itemID, ClientVersionCode, header) cm := NewCM() resp, err := GorgonRequest(backupAPI, map[string]interface{}{ @@ -259,18 +292,14 @@ func tryBackupAPI(itemID string, textMode, imageMode bool) (string, error) { contentLogger.Error("解密失败: " + err.Error()) return `{"code": -1, "msg": "解密失败"}`, err } + decryptedContent = string(gunzipMaybe([]byte(decryptedContent))) - data["content"] = decryptedContent + data["content"] = strings.TrimSpace(decryptedContent) - // 更新缓存 - bodyJSONStr := buildJSONResponse(body) - UpdateCache("content_"+itemID, bodyJSONStr, 86400) - - // 解析内容 - parser := NewContentParser() - data["content"] = parser.Parse(decryptedContent, textMode, imageMode) - - return buildJSONResponse(UnwrapJSONStrings(body)), nil + // 写缓存并返回缓存格式 + cacheStr := marshalNoEscape(buildCacheData(UnwrapJSONStrings(data))) + UpdateCache("content_"+itemID, cacheStr, 86400) + return cacheStr, nil } contentLogger.Warning("备用接口1获取正文出错！") diff --git a/internal/fqapi/content_parser.go b/internal/fqapi/content_parser.go index 400dbe4..b4e2f39 100644 --- a/internal/fqapi/content_parser.go +++ b/internal/fqapi/content_parser.go @@ -9,6 +9,20 @@ import ( "github.com/PuerkitoBio/goquery" ) +// imgSrcRe 匹配 img 标签的 src 属性 +var imgSrcRe = regexp.MustCompile(`(?i)(]*\ssrc=")([^"]*)(")`) + +// permanentizeImgSrc 将正文插图链接替换为永久 CDN 链接（原链接带临时参数会过期） +func permanentizeImgSrc(html string) string { + return imgSrcRe.ReplaceAllStringFunc(html, func(m string) string { + parts := imgSrcRe.FindStringSubmatch(m) + if len(parts) < 3 || parts[2] == "" { + return m + } + return parts[1] + NormalizeNovelImageURL(parts[2]) + parts[3] + }) +} + var ( parserLogger = common.NewLogger("content_parser", "INFO", "") ) @@ -118,6 +132,8 @@ func (p *ContentParser) extractText(content string) string { divHTML = decodeHTMLEntities(divHTML) // 标准化img属性 divHTML = normalizeImgAttributes(divHTML) + // 插图链接永久化 + divHTML = permanentizeImgSrc(divHTML) finalResult = append(finalResult, divHTML) } return @@ -173,6 +189,9 @@ func (p *ContentParser) processHTMLMode(content string) string { // 解码HTML实体 content = decodeHTMLEntities(content) + // 插图链接永久化（原链接带临时参数会过期） + content = permanentizeImgSrc(content) + // 根据配置决定是否移除空行 if p.cfg.RemoveBlankLines { // 对HTML内容移除空的

标签 diff --git a/internal/fqapi/content_utils.go b/internal/fqapi/content_utils.go index fa6370b..9f9d88d 100644 --- a/internal/fqapi/content_utils.go +++ b/internal/fqapi/content_utils.go @@ -2,12 +2,84 @@ package fqapi import ( "bytes" + "compress/gzip" + "compress/zlib" "encoding/json" "fanqie/internal/config" - "sort" + "io" + "net/url" + "regexp" "strings" ) +// 客户端版本统一（与 2026-10 可用的书源实现一致；旧值 58932/5.8.9.32 混用易触发风控） +const ( + ClientVersionCode = "72732" + ClientVersionName = "7.2.7.32" +) + +// PermanentNovelImgHost 番茄小说图片的永久 CDN 域名（原始链接带临时参数会过期失效） +const PermanentNovelImgHost = "https://p6-novel.byteimg.com" + +// novelPicMd5Re 匹配图片路径中的 32 位十六进制资源 ID +var novelPicMd5Re = regexp.MustCompile(`^/novel-pic/([a-f0-9]{32})`) + +// NormalizeNovelImageURL 将临时图片链接永久化： +// - /novel-pic/{md5} → p6-novel.byteimg.com/origin/novel-pic/{md5}（正文插图原图） +// - 含 ~tplv 的模板图（封面缩放）→ p6-novel.byteimg.com 保留原路径 +func NormalizeNovelImageURL(raw string) string { + u := strings.TrimSpace(strings.ReplaceAll(raw, "&", "&")) + if u == "" { + return "" + } + if strings.HasPrefix(u, "//") { + u = "https:" + u + } + idx := strings.Index(u, "/novel-pic/") + if idx < 0 { + return u + } + path := u[idx:] + if m := novelPicMd5Re.FindStringSubmatch(path); m != nil && !strings.Contains(path, "~tplv") { + return PermanentNovelImgHost + "/origin/novel-pic/" + m[1] + } + return PermanentNovelImgHost + path +} + +// gunzipMaybe 自适应解压上游正文（gzip 与 zlib 帧都可能出现），非压缩数据原样返回 +func gunzipMaybe(data []byte) []byte { + if r, err := gzip.NewReader(bytes.NewReader(data)); err == nil { + if out, err2 := io.ReadAll(r); err2 == nil { + r.Close() + return out + } + } + if r, err := zlib.NewReader(bytes.NewReader(data)); err == nil { + if out, err2 := io.ReadAll(r); err2 == nil { + r.Close() + return out + } + } + return data +} + +// buildSortedQuery 将若干 "k=v" 段合并为 form 编码的查询串（按键排序）。 +// 设备机型等值含空格时必须编码，否则实际请求的 URL 与签名输入不一致， +// registerkey 等接口会注册失败（书源 2026-10-10 修复的同类问题） +func buildSortedQuery(parts ...string) string { + vals := url.Values{} + for _, part := range parts { + for _, seg := range strings.Split(part, "&") { + if seg == "" { + continue + } + k, v, _ := strings.Cut(seg, "=") + vals.Add(k, v) + } + } + return vals.Encode() +} + // getContentConfig 获取内容配置（带默认值） func getContentConfig() config.ContentConfig { if config.GetAppConfig() != nil { @@ -188,7 +260,7 @@ func getDeviceHeader() string { // removeVersionFromHeader 移除header中的version_name参数 func removeVersionFromHeader(header string) string { - return strings.Replace(header, "&version_name=5.8.9.32", "", -1) + return strings.Replace(header, "&version_name="+ClientVersionName, "", -1) } // ============ API参数构建工具 ============ @@ -203,7 +275,7 @@ func BuildURL(fullURL string, customParams ...string) string { "channel=0", "device_platform=android", "os_version=0", - "version_code=58932", + "version_code=" + ClientVersionCode, } // 添加设备header @@ -215,13 +287,8 @@ func BuildURL(fullURL string, customParams ...string) string { // 添加自定义参数 allParams = append(allParams, customParams...) - // 合并、拆分、排序 - joined := strings.Join(allParams, "&") - paramList := strings.Split(joined, "&") - sort.Strings(paramList) - params := strings.Join(paramList, "&") - - return fullURL + "?" + params + // form 编码 + 排序（机型含空格等特殊字符时保证 URL 与签名一致） + return fullURL + "?" + buildSortedQuery(allParams...) } // ============ 缓存处理工具函数 ============ diff --git a/internal/fqapi/content_utils_test.go b/internal/fqapi/content_utils_test.go new file mode 100644 index 0000000..bbdfd25 --- /dev/null +++ b/internal/fqapi/content_utils_test.go @@ -0,0 +1,31 @@ +package fqapi + +import "testing" + +// TestBuildSortedQuery form 编码 + 排序（机型含空格时 URL 与签名一致的关键） +func TestBuildSortedQuery(t *testing.T) { + got := buildSortedQuery("b=2", "a=1", "device_type=Pixel 6") + want := "a=1&b=2&device_type=Pixel+6" + if got != want { + t.Fatalf("buildSortedQuery = %q, want %q", got, want) + } +} + +// TestNormalizeNovelImageURL 插图/封面/已永久化链接的归一化 +func TestNormalizeNovelImageURL(t *testing.T) { + cases := map[string]string{ + // 正文插图（纯 md5）→ origin 原图 + "//p9-novel.byteimg.com/img/novel-pic/0123456789abcdef0123456789abcdef": "https://p6-novel.byteimg.com/origin/novel-pic/0123456789abcdef0123456789abcdef", + // 已永久化的链接幂等 + "https://p6-novel.byteimg.com/origin/novel-pic/0123456789abcdef0123456789abcdef": "https://p6-novel.byteimg.com/origin/novel-pic/0123456789abcdef0123456789abcdef", + // 封面模板图（~tplv 缩放）→ 保留原路径仅换域名 + "https://p3-novel.byteimg.com/novel-pic/0123456789abcdef0123456789abcdef~tplv-novel-pic-large.webp": "https://p6-novel.byteimg.com/novel-pic/0123456789abcdef0123456789abcdef~tplv-novel-pic-large.webp", + // 非 novel-pic 链接原样返回 + "https://example.com/other.jpg": "https://example.com/other.jpg", + } + for in, want := range cases { + if got := NormalizeNovelImageURL(in); got != want { + t.Errorf("NormalizeNovelImageURL(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/fqapi/detail.go b/internal/fqapi/detail.go index c998bc8..00663d3 100644 --- a/internal/fqapi/detail.go +++ b/internal/fqapi/detail.go @@ -42,9 +42,13 @@ func fetchDetail(bookID string) (string, error) { } if code, ok := respJSON["code"].(float64); ok && code == 0 { - // 删除content字段 if data, ok := respJSON["data"].(map[string]interface{}); ok { + // 删除content字段 delete(data, "content") + // 封面链接永久化（原链接带临时参数会过期） + if thumb, ok := data["thumb_url"].(string); ok && thumb != "" { + data["thumb_url"] = NormalizeNovelImageURL(thumb) + } } // 重新序列化（解包 JSON 字符串字段） diff --git a/internal/fqapi/device_key_pool.go b/internal/fqapi/device_key_pool.go index 1ecef7e..8e366fe 100644 --- a/internal/fqapi/device_key_pool.go +++ b/internal/fqapi/device_key_pool.go @@ -20,9 +20,14 @@ type DeviceKeyItem struct { DeviceHeader string `json:"device_header"` Key string `json:"key"` KeyVer int `json:"key_ver"` + KeyTs int64 `json:"key_ts"` // 密钥注册时间戳（key_register_ts） IsValid bool `json:"is_valid"` CreatedAt time.Time `json:"created_at"` LastUsedAt time.Time `json:"last_used_at"` + + // 仅内存字段（不落库） + KeyFetchedAt time.Time `json:"-"` // 密钥获取时间，超 24h 异步刷新 + KeyRefreshAt time.Time `json:"-"` // 上次刷新尝试时间，防刷新风暴 } // DeviceKeyPool 设备密钥池 @@ -77,6 +82,10 @@ func (p *DeviceKeyPool) Initialize() error { } poolLogger.Info(fmt.Sprintf("设备密钥池初始化完成，当前有效设备数: %d", p.getValidCount())) + + // last_used_at 批量落库（GetNext 不再同步写库） + p.StartDevicePoolFlusher() + return nil } @@ -110,11 +119,22 @@ func (p *DeviceKeyPool) GetNext() (*DeviceKeyItem, error) { item := validItems[p.currentIndex%len(validItems)] p.currentIndex++ - // 更新最后使用时间 + // 只更新内存中的最后使用时间，由后台 flusher 定期批量落库 + // （此前每次取设备同步 UPDATE，SQLite 单写锁下是吞吐瓶颈） item.LastUsedAt = time.Now() - p.updateDeviceInDB(item) - // poolLogger.Debug(fmt.Sprintf("使用设备 ID=%d", item.ID)) + // 密钥超 24h 未刷新：异步换新（本次请求仍用旧密钥，下次生效） + if time.Since(item.KeyFetchedAt) > 20*time.Hour && time.Since(item.KeyRefreshAt) > 10*time.Minute { + item.KeyRefreshAt = time.Now() + header := item.DeviceHeader + go func() { + if ki, err := GetKey(header); err == nil { + UpdateKeyInPool(header, ki) + } else { + poolLogger.Warning(fmt.Sprintf("刷新设备密钥失败: %v", err)) + } + }() + } // 异步检查是否需要补充 go p.checkAndRefill() @@ -316,9 +336,11 @@ func (p *DeviceKeyPool) addNewDevice() error { DeviceHeader: deviceHeader, Key: keyInfo.Key, KeyVer: keyInfo.Ver, + KeyTs: keyInfo.Ts, IsValid: true, CreatedAt: time.Now(), LastUsedAt: time.Now(), + KeyFetchedAt: time.Now(), } // 保存到数据库 @@ -344,8 +366,8 @@ func (p *DeviceKeyPool) loadFromDB() error { } rows, err := cacheDB.Query(` - SELECT id, device_header, key, key_ver, is_valid, created_at, last_used_at - FROM device_pool + SELECT id, device_header, key, key_ver, COALESCE(key_ts, 0), is_valid, created_at, last_used_at + FROM device_pool ORDER BY id `) if err != nil { @@ -363,6 +385,7 @@ func (p *DeviceKeyPool) loadFromDB() error { &item.DeviceHeader, &item.Key, &item.KeyVer, + &item.KeyTs, &item.IsValid, &createdAt, &lastUsedAt, @@ -374,6 +397,7 @@ func (p *DeviceKeyPool) loadFromDB() error { item.CreatedAt = time.Unix(createdAt, 0) item.LastUsedAt = time.Unix(lastUsedAt, 0) + item.KeyFetchedAt = time.Now() // 旧设备的密钥年龄未知，从加载起算 items = append(items, &item) } @@ -401,10 +425,10 @@ func (p *DeviceKeyPool) saveDeviceToDB(item *DeviceKeyItem) error { // RETURNING id 同时兼容 SQLite(3.35+) 与 PostgreSQL，替代 LastInsertId() err := cacheDB.QueryRow(q(` - INSERT INTO device_pool (device_header, key, key_ver, is_valid, created_at, last_used_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO device_pool (device_header, key, key_ver, key_ts, is_valid, created_at, last_used_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id - `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), + `), item.DeviceHeader, item.Key, item.KeyVer, item.KeyTs, boolToInt(item.IsValid), item.CreatedAt.Unix(), item.LastUsedAt.Unix(), item.CreatedAt.Unix()).Scan(&item.ID) return err @@ -418,14 +442,39 @@ func (p *DeviceKeyPool) updateDeviceInDB(item *DeviceKeyItem) error { _, err := cacheDB.Exec(q(` UPDATE device_pool - SET device_header = ?, key = ?, key_ver = ?, is_valid = ?, last_used_at = ?, updated_at = ? + SET device_header = ?, key = ?, key_ver = ?, key_ts = ?, is_valid = ?, last_used_at = ?, updated_at = ? WHERE id = ? - `), item.DeviceHeader, item.Key, item.KeyVer, boolToInt(item.IsValid), + `), item.DeviceHeader, item.Key, item.KeyVer, item.KeyTs, boolToInt(item.IsValid), item.LastUsedAt.Unix(), item.LastUsedAt.Unix(), item.ID) return err } +// StartDevicePoolFlusher 启动 last_used_at 批量落库任务（GetNext 已改为只写内存） +func (p *DeviceKeyPool) StartDevicePoolFlusher() { + go func() { + for range time.Tick(time.Minute) { + FlushDevicePool() + } + }() + poolLogger.Info("设备池 last_used_at 批量落库已启动（每分钟）") +} + +// FlushDevicePool 将内存中的设备使用时间批量刷入数据库 +// ponytail: 进程退出时最多丢 1 分钟使用记录（仅统计用途），无需精确停表 +func FlushDevicePool() { + p := GetDeviceKeyPool() + if !p.enabled || cacheDB == nil { + return + } + p.mu.RLock() + items := append([]*DeviceKeyItem(nil), p.items...) + p.mu.RUnlock() + for _, it := range items { + p.updateDeviceInDB(it) + } +} + // removeInvalidFromDB 从数据库移除无效设备 func (p *DeviceKeyPool) removeInvalidFromDB() error { if cacheDB == nil { @@ -483,10 +532,10 @@ func GetKeyInfoFromPool(deviceHeader string) *KeyInfo { return nil } -// UpdateKeyInPool 更新池中设备的密钥 -func UpdateKeyInPool(deviceHeader string, key string, ver int) error { +// UpdateKeyInPool 更新池中设备的密钥（含注册时间戳） +func UpdateKeyInPool(deviceHeader string, ki *KeyInfo) error { pool := GetDeviceKeyPool() - if !pool.IsEnabled() { + if !pool.IsEnabled() || ki == nil { return nil } @@ -495,8 +544,10 @@ func UpdateKeyInPool(deviceHeader string, key string, ver int) error { for _, item := range pool.items { if item.DeviceHeader == deviceHeader { - item.Key = key - item.KeyVer = ver + item.Key = ki.Key + item.KeyVer = ki.Ver + item.KeyTs = ki.Ts + item.KeyFetchedAt = time.Now() return pool.updateDeviceInDB(item) } } diff --git a/internal/fqapi/device_register.go b/internal/fqapi/device_register.go index c2002d5..ac76383 100644 --- a/internal/fqapi/device_register.go +++ b/internal/fqapi/device_register.go @@ -252,9 +252,9 @@ func tryActivatePremium(header string) { "channel=0", "device_platform=android", "os_version=0", - "version_code=58932", - "manifest_version_code=58932", - "update_version_code=58932", + "version_code=" + ClientVersionCode, + "manifest_version_code=" + ClientVersionCode, + "update_version_code=" + ClientVersionCode, header, } @@ -365,7 +365,7 @@ func Register() (string, error) { } deviceModel := data["header"].(map[string]interface{})["device_model"].(string) - header := fmt.Sprintf("device_id=%v&device_type=%s&iid=%v&version_name=5.8.9.32", + header := fmt.Sprintf("device_id=%v&device_type=%s&iid=%v&version_name="+ClientVersionName, respData["device_id_str"], deviceModel, respData["install_id_str"]) diff --git a/internal/fqapi/gorgon_utils.go b/internal/fqapi/gorgon_utils.go index 593f121..3573c93 100644 --- a/internal/fqapi/gorgon_utils.go +++ b/internal/fqapi/gorgon_utils.go @@ -14,6 +14,16 @@ func CreateMD5(s string) string { return hex.EncodeToString(hasher.Sum(nil)) } +// setCommonHeaders 注入番茄客户端通用请求头（与签名头不冲突，对齐书源实现降低风控特征） +func setCommonHeaders(headers map[string]string) { + if _, ok := headers["x-tt-store-region"]; ok { + return + } + headers["x-tt-store-region"] = "cn-zj" + headers["x-tt-store-region-src"] = "did" + headers["sdk-version"] = "2" +} + // GorgonRequest Gorgon请求（使用签名切换器） func GorgonRequest(urlStr string, options map[string]interface{}) (*common.ClientResponse, error) { headers, ok := options["headers"].(map[string]string) @@ -21,6 +31,7 @@ func GorgonRequest(urlStr string, options map[string]interface{}) (*common.Clien headers = make(map[string]string) options["headers"] = headers } + setCommonHeaders(headers) dataMD5 := "" if body, ok := options["body"].(string); ok && body != "" { @@ -53,6 +64,7 @@ func GorgonRequestSync(urlStr string, options map[string]interface{}) (*common.C headers = make(map[string]string) options["headers"] = headers } + setCommonHeaders(headers) dataMD5 := "" if body, ok := options["body"].(string); ok && body != "" { @@ -69,9 +81,7 @@ func GorgonRequestSync(urlStr string, options map[string]interface{}) (*common.C signedHeaders := algos.GlobalSignatureSwitcher.Sign(urlStr, headers) // 删除x-ss-stub（如果存在） - if _, exists := signedHeaders["x-ss-stub"]; exists { - delete(signedHeaders, "x-ss-stub") - } + delete(signedHeaders, "x-ss-stub") // 更新headers for k, v := range signedHeaders { @@ -90,6 +100,7 @@ func ContentRequest(urlStr string, options map[string]interface{}) (*common.Clie headers = make(map[string]string) options["headers"] = headers } + setCommonHeaders(headers) dataMD5 := "" if body, ok := options["body"].(string); ok && body != "" { diff --git a/internal/fqapi/manga.go b/internal/fqapi/manga.go index 2d1bf0a..d1c03c6 100644 --- a/internal/fqapi/manga.go +++ b/internal/fqapi/manga.go @@ -87,7 +87,7 @@ func getMangaWithRetry(itemID string, showHTML bool, urlOutputMode string, retry mangaLogger.Debug(fmt.Sprintf("使用池中设备 ID=%d (重试次数: %d)", item.ID, retryCount)) // 2. 构建URL - 使用 BuildURL 添加设备参数 - headerWithoutVersion := strings.Replace(header, "&version_name=5.8.9.32", "", -1) + headerWithoutVersion := removeVersionFromHeader(header) url := BuildURL("https://reading.snssdk.com/reading/reader/full/v/", "aid=1967", @@ -146,7 +146,7 @@ func getMangaWithRetry(itemID string, showHTML bool, urlOutputMode string, retry mangaLogger.Warning("密钥版本不匹配，重新获取密钥") newKeyInfo, err := GetKey(header) if err == nil { - UpdateKeyInPool(header, newKeyInfo.Key, newKeyInfo.Ver) + UpdateKeyInPool(header, newKeyInfo) keyInfo = newKeyInfo } else { mangaLogger.Warning("获取密钥失败，切换设备重试") diff --git a/internal/fqapi/multi_content.go b/internal/fqapi/multi_content.go index 8bb2692..3e63e17 100644 --- a/internal/fqapi/multi_content.go +++ b/internal/fqapi/multi_content.go @@ -1,14 +1,9 @@ package fqapi import ( - "bytes" - "compress/zlib" - "encoding/base64" "encoding/json" "fanqie/internal/common" "fmt" - "io" - "sort" "strings" "sync" ) @@ -17,57 +12,29 @@ var ( multiContentLogger = common.NewLogger("multi_content", "INFO", "") ) -// handleDecryptTask 处理解密任务 +// handleDecryptTask 处理解密任务（crypt_status 分支见 decodeContentBody） func handleDecryptTask(itemID string, itemData map[string]interface{}, keyInfo *KeyInfo, textMode, imageMode bool) (string, map[string]interface{}) { if code, ok := itemData["code"].(float64); !ok || code != 0 { return itemID, itemData } - contentEncoded, ok := itemData["content"].(string) - if !ok { + if _, ok := itemData["content"].(string); !ok { return itemID, itemData } - // 解码content - contentBytes, err := base64.StdEncoding.DecodeString(contentEncoded) - if err != nil || len(contentBytes) < 16 { + // 解密（含压缩与明文分支） + contentStr, err := decodeContentBody(itemData, keyInfo) + if err != nil || contentStr == "" { + // 解密失败保留上游原样（ignore_failed 默认跳过失败章节） return itemID, itemData } - iv := contentBytes[0:16] - encryptedData := contentBytes[16:] - - // 解密 - if keyInfo == nil { - return itemID, itemData - } - - keyBytes, _ := base64.StdEncoding.DecodeString(keyInfo.Key) - decrypted, err := De(encryptedData, keyBytes, iv) - if err != nil { - return itemID, itemData - } - - // 检查是否需要解压 - compressStatus, _ := itemData["compress_status"].(float64) - if compressStatus != 0 { - // zlib解压 - reader, err := zlib.NewReader(bytes.NewReader(decrypted)) - if err == nil { - decompressed, err := io.ReadAll(reader) - reader.Close() - if err == nil { - decrypted = decompressed - } - } - } - // 更新内容 bodyCopy := make(map[string]interface{}) for k, v := range itemData { bodyCopy[k] = v } - bodyCopy["content"] = string(decrypted) + bodyCopy["content"] = contentStr // 缓存 cacheData := buildCacheData(bodyCopy) @@ -75,7 +42,7 @@ func handleDecryptTask(itemID string, itemData map[string]interface{}, keyInfo * // 解析内容 parser := NewContentParser() - bodyCopy["content"] = parser.Parse(string(decrypted), textMode, imageMode) + bodyCopy["content"] = parser.Parse(contentStr, textMode, imageMode) return itemID, bodyCopy } @@ -121,10 +88,11 @@ func MultiContent(itemIDs interface{}, bookID string, textMode, imageMode, ignor currentKeyInfo := &KeyInfo{ Key: item.Key, Ver: item.KeyVer, + Ts: item.KeyTs, } multiContentLogger.Debug(fmt.Sprintf("使用池中设备 ID=%d (depth: %d)", item.ID, depth)) - // 构建参数 + // 构建参数（form 编码，见 buildSortedQuery 注释） headerWithoutVersion := removeVersionFromHeader(header) params := []string{ @@ -133,18 +101,15 @@ func MultiContent(itemIDs interface{}, bookID string, textMode, imageMode, ignor "channel=0", "device_platform=android", "os_version=0", - "version_code=66.9", + "version_code=" + ClientVersionCode, "book_id=" + bookID, "item_ids=" + itemIDStr, "novel_text_type=1", "req_type=1", + fmt.Sprintf("key_register_ts=%d", currentKeyInfo.Ts), headerWithoutVersion, } - - allParams := strings.Join(params, "&") - paramList := strings.Split(allParams, "&") - sort.Strings(paramList) - sortedParams := strings.Join(paramList, "&") + sortedParams := buildSortedQuery(params...) url := "https://reading.snssdk.com/reading/reader/batch_full/v/?" + sortedParams @@ -205,7 +170,7 @@ func MultiContent(itemIDs interface{}, bookID string, textMode, imageMode, ignor multiContentLogger.Warning("密钥版本不匹配，重新获取密钥") newKeyInfo, err := GetKey(header) if err == nil { - UpdateKeyInPool(header, newKeyInfo.Key, newKeyInfo.Ver) + UpdateKeyInPool(header, newKeyInfo) } else { // 获取密钥失败，切换设备 multiContentLogger.Warning(fmt.Sprintf("获取密钥失败，切换设备重试 (depth: %d)", depth)) diff --git a/internal/fqapi/multi_detail.go b/internal/fqapi/multi_detail.go index c5ff2d3..baf3e80 100644 --- a/internal/fqapi/multi_detail.go +++ b/internal/fqapi/multi_detail.go @@ -3,6 +3,7 @@ package fqapi import ( "fanqie/internal/common" "fmt" + "sort" "strings" "sync" ) @@ -108,11 +109,41 @@ func chunkArray(arr []string, size int) [][]string { // HandleGetMultiDetail 处理批量获取书籍详情,详情数据基本不变,缓存 24 小时 func HandleGetMultiDetail(bookID interface{}) (string, error) { - return withCache(fmt.Sprintf("multi_detail_%v", bookID), 86400, func() (string, error) { + // 缓存 key 归一化：ID 排序去重，相同集合不同顺序共享缓存 + ids := normalizeIDList(bookID) + if ids == "" { + return fetchMultiDetail(bookID) + } + return withCache("multi_detail_"+ids, 86400, func() (string, error) { return fetchMultiDetail(bookID) }) } +// normalizeIDList 将 book_id 参数归一化为排序去重后的逗号串 +func normalizeIDList(bookID interface{}) string { + var list []string + switch v := bookID.(type) { + case string: + list = strings.Split(v, ",") + case []string: + list = v + default: + return "" + } + seen := make(map[string]bool, len(list)) + out := make([]string, 0, len(list)) + for _, id := range list { + id = strings.TrimSpace(id) + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, id) + } + sort.Strings(out) + return strings.Join(out, ",") +} + func fetchMultiDetail(bookID interface{}) (string, error) { if bookID == nil { return BuildErrorResponse("需要参数 book_id"), nil diff --git a/internal/fqapi/rank.go b/internal/fqapi/rank.go new file mode 100644 index 0000000..67dadb3 --- /dev/null +++ b/internal/fqapi/rank.go @@ -0,0 +1,77 @@ +package fqapi + +import ( + "fanqie/internal/common" + "fmt" + "net/url" +) + +var rankLogger = common.NewLogger("rank", "INFO", "") + +// rankSDKHosts 免签名榜单接口的可用域名（依次容错） +var rankSDKHosts = []string{ + "https://novel.snssdk.com", + "https://api.fanqiesdk.com", + "https://api-lf.fanqiesdk.com", +} + +const rankSDKUA = "com.ss.android.article.news/7.0.3 (Linux; U; Android 12; zh_CN; Pixel 6)" + +// RankTypes 榜单类型 +var RankTypes = map[int]string{ + 10: "推荐榜", + 11: "完结榜", + 12: "热搜榜", + 13: "黑马榜", + 15: "爆更榜", +} + +// HandleGetRank 处理获取榜单,榜单数据动态,短缓存 10 分钟 +// sideType 见 RankTypes（无效值回退推荐榜）；page 从 1 起，每页 20 本 +func HandleGetRank(sideType, page int) (string, error) { + if _, ok := RankTypes[sideType]; !ok { + sideType = 10 + } + if page < 1 { + page = 1 + } + return withCache(fmt.Sprintf("rank_%d_%d", sideType, page), 600, func() (string, error) { + return fetchRank(sideType, page) + }) +} + +// fetchRank 免签名 SDK 接口获取榜单（无设备/签名依赖，多域名容错） +func fetchRank(sideType, page int) (string, error) { + vals := url.Values{ + "aid": {"13"}, + "type": {"1"}, + "side_type": {fmt.Sprintf("%d", sideType)}, + "offset": {fmt.Sprintf("%d", page-1)}, + "limit": {"20"}, + } + path := "/api/novel/channel/homepage/rank/rank_list/v2/?" + vals.Encode() + + var lastErr error + client := common.NewHTTPClient() + for _, host := range rankSDKHosts { + resp, err := client.Get(host+path, map[string]string{ + "User-Agent": rankSDKUA, + "Accept": "application/json, text/plain, */*", + }) + if err != nil { + lastErr = err + continue + } + respJSON, err := resp.JSON() + if err != nil { + lastErr = fmt.Errorf("响应解析失败: %w", err) + continue + } + if code, ok := respJSON["code"].(float64); ok && code == 0 { + return reencodeUnwrapped(resp.Text), nil + } + lastErr = fmt.Errorf("code=%.0f msg=%v", respJSON["code"], respJSON["message"]) + } + rankLogger.Error(fmt.Sprintf("获取榜单失败: %v", lastErr)) + return BuildErrorResponse("获取榜单失败"), lastErr +} diff --git a/internal/fqapi/register_key.go b/internal/fqapi/register_key.go index 164c050..39eb14f 100644 --- a/internal/fqapi/register_key.go +++ b/internal/fqapi/register_key.go @@ -12,15 +12,13 @@ import ( "fmt" "math/rand" "regexp" - "sort" - "strings" - "time" ) // KeyInfo 密钥信息 type KeyInfo struct { Key string `json:"key"` Ver int `json:"ver"` + Ts int64 `json:"ts"` // 密钥注册时间戳（key_register_ts），随正文请求上传可降低密钥不匹配重试 } var ( @@ -154,25 +152,22 @@ func GetKey(header string) (*KeyInfo, error) { keyLogger.Info("正在获取密钥...") - // 构建参数 + // 构建参数（form 编码：机型含空格等字符时 URL 与签名一致才不会注册失败） params := []string{ "aid=1967", "app_name=novelapp", "channel=0", "device_platform=android", "os_version=0", - "version_code=5.8.9.32", + "version_code=" + ClientVersionCode, header, } - - allParams := strings.Join(params, "&") - paramList := strings.Split(allParams, "&") - sort.Strings(paramList) - sortedParams := strings.Join(paramList, "&") + sortedParams := buildSortedQuery(params...) // 准备请求体 requestBody := map[string]string{ "content": content, + "keyver": "1", } bodyJSON, _ := json.Marshal(requestBody) @@ -203,6 +198,12 @@ func GetKey(header string) (*KeyInfo, error) { var respData map[string]interface{} json.Unmarshal(resp.Content, &respData) + // 上游判定该设备无需解密（正文明文下发），返回空密钥信息由调用方分支处理 + if code, ok := respData["code"].(float64); ok && int(code) == 500003 { + keyLogger.Info("上游标记无需解密（code=500003）") + return &KeyInfo{}, nil + } + dataObj, ok := respData["data"].(map[string]interface{}) if !ok { return nil, fmt.Errorf("响应格式错误") @@ -231,12 +232,17 @@ func GetKey(header string) (*KeyInfo, error) { finalKey := base64.StdEncoding.EncodeToString(decryptedKey) keyVer, _ := dataObj["keyver"].(float64) + keyTs, _ := dataObj["key_register_ts"].(float64) + if keyTs == 0 { + keyTs, _ = dataObj["register_time"].(float64) + } keyInfo = &KeyInfo{ Key: finalKey, Ver: int(keyVer), + Ts: int64(keyTs), } - keyLogger.Info(fmt.Sprintf("密钥获取成功！")) + keyLogger.Info("密钥获取成功！") return keyInfo, nil } @@ -248,7 +254,3 @@ func GetKey(header string) (*KeyInfo, error) { func GetKeyInfo() *KeyInfo { return keyInfo } - -func init() { - rand.Seed(time.Now().UnixNano()) -} diff --git a/internal/fqapi/request_log.go b/internal/fqapi/request_log.go index 4f9b1e7..5210e72 100644 --- a/internal/fqapi/request_log.go +++ b/internal/fqapi/request_log.go @@ -16,7 +16,7 @@ type RequestLogEntry struct { IP string `json:"ip"` Method string `json:"method"` Path string `json:"path"` - Query string `json:"query"` // 解码后查询串（含 key，不脱敏） + Query string `json:"query"` // 解码后查询串（key 参数值已脱敏） Status int `json:"status"` DurationMs int64 `json:"duration_ms"` ErrorMsg string `json:"error_msg"` // 仅 status>=400 时截断保留 diff --git a/internal/fqapi/search.go b/internal/fqapi/search.go index 52f81a6..5ad4a0e 100644 --- a/internal/fqapi/search.go +++ b/internal/fqapi/search.go @@ -6,12 +6,22 @@ import ( "fanqie/internal/common" "fmt" "net/url" + "regexp" + "strconv" + "time" ) var searchLogger = common.NewLogger("search", "INFO", "") -// GetDeviceHeader 获取设备header（需要从外部注入） -var GetDeviceHeader func() string +// 搜索分页与上游续传参数（对齐书源 20261012：上游翻页依赖上一页返回的 +// passback/search_id/next_offset/corrected_query 续传，仅传数字 offset 深翻页会拿到重复/错序结果） +const ( + searchPageSize = 10 // 对外每页书数 + searchMaxUpstreams = 24 // 单次搜索累计上游翻页上限（对齐书源） +) + +// bookIDKeyRe 纯数字 15~20 位关键词按书 ID 直搜详情 +var bookIDKeyRe = regexp.MustCompile(`^\d{15,20}$`) // HandleBookSearch 处理书籍搜索,搜索结果短缓存 10 分钟,减轻上游压力 func HandleBookSearch(key string, offset int, tabType int) (string, error) { @@ -24,21 +34,111 @@ func fetchBookSearch(key string, offset int, tabType int) (string, error) { if key == "" { return BuildErrorResponseWithCode(1, "需要参数 key"), nil } - if tabType == 0 { tabType = 3 } + if offset < 0 { + offset = 0 + } - // 编码查询关键字 - encodedKey := url.QueryEscape(key) + // 书 ID 直搜：跳过搜索接口直接查详情，精确命中单本 + if bookIDKeyRe.MatchString(key) { + return idSearchResult(key) + } - // 使用新的参数构建函数 - reqURL := BuildURL("https://reading.snssdk.com/reading/bookapi/search/tab/v/", - fmt.Sprintf("tab_type=%d", tabType), - "update_version_code=58932", - "query="+encodedKey, - fmt.Sprintf("passback=%d", offset), + // 累计翻页直到凑够本页书：上游单页 count=20，按 book_id 去重后切出 offset 起的 10 本 + session := strconv.FormatInt(time.Now().UnixNano(), 10) + var ( + rows []interface{} // 上游搜索行（含 book_data 包装），原样透传 + seen = map[string]bool{} + passback = "0" + searchID = "" + corrected = "" + nextOff = 0 + hasMore = true ) + want := offset + searchPageSize + + for i := 0; i < searchMaxUpstreams && hasMore && len(rows) < want; i++ { + params := []string{ + fmt.Sprintf("tab_type=%d", tabType), + "update_version_code=" + ClientVersionCode, + "query=" + url.QueryEscape(key), + "passback=" + passback, + fmt.Sprintf("offset=%d", nextOff), + "count=20", + fmt.Sprintf("is_first_enter_search=%t", i == 0), + "normal_session_id=" + session, + "cold_start_session_id=" + session, + } + if searchID != "" { + params = append(params, "search_id="+searchID) + } + if corrected != "" { + params = append(params, "corrected_query="+corrected) + } + + tab, err := fetchSearchTab(params, tabType) + if err != nil { + if i == 0 { + return BuildErrorResponseWithCode(1, "搜索请求失败"), err + } + break // 已有数据时后续页失败，降级返回现有结果 + } + if tab == nil { + break + } + + if data, ok := tab["data"].([]interface{}); ok { + for _, r := range data { + row, _ := r.(map[string]interface{}) + if row == nil { + continue + } + // 行 ID 取 book_data[0].book_id；无包装的行按行本身的 book_id + bd, _ := row["book_data"].([]interface{}) + if len(bd) > 0 { + if b, ok := bd[0].(map[string]interface{}); ok { + if id := searchRowID(b); id != "" && !seen[id] { + seen[id] = true + rows = append(rows, row) + } + continue + } + } + if id := searchRowID(row); id != "" && !seen[id] { + seen[id] = true + rows = append(rows, row) + } + } + } + + hasMore = searchHasMore(tab["has_more"]) + nextOff = searchInt(tab["next_offset"]) + passback = searchStr(tab["passback"]) + searchID = searchStr(tab["search_id"]) + corrected = searchStr(tab["corrected_query"]) + } + + start, end := offset, offset+searchPageSize + if start > len(rows) { + start = len(rows) + } + if end > len(rows) { + end = len(rows) + } + return marshalNoEscape(UnwrapJSONStrings(map[string]interface{}{ + "code": 0, + "message": "SUCCESS", + "data": rows[start:end], + "has_more": hasMore || end < len(rows), + "next_offset": offset + searchPageSize, + })), nil +} + +// fetchSearchTab 请求一页搜索并返回目标 tab_type 的 tab；nil 表示该页无匹配 tab +func fetchSearchTab(params []string, tabType int) (map[string]interface{}, error) { + reqURL := BuildURL("https://reading.snssdk.com/reading/bookapi/search/tab/v/", params...) resp, err := GorgonRequest(reqURL, map[string]interface{}{ "method": "GET", @@ -46,35 +146,61 @@ func fetchBookSearch(key string, offset int, tabType int) (string, error) { "User-Agent": "com.dragon.read", }, }) - if err != nil { searchLogger.Error(fmt.Sprintf("搜索请求失败: %v", err)) - return BuildErrorResponseWithCode(1, "搜索请求失败"), err + return nil, err } - respJSON, err := resp.JSON() - if err != nil { + // UseNumber 重新解析后按 tab_type 过滤: + // 上游 search/tab/v/ 固定返回所有 tab 的 search_tabs; + // 大整数 ID 经 float64 重组会丢精度,必须 UseNumber + dec := json.NewDecoder(bytes.NewReader(resp.Content)) + dec.UseNumber() + var pageJSON map[string]interface{} + if err := dec.Decode(&pageJSON); err != nil { searchLogger.Error(fmt.Sprintf("解析响应失败: %v", err)) - return BuildErrorResponseWithCode(1, "解析响应失败"), err + return nil, fmt.Errorf("解析失败") + } + if code, _ := pageJSON["code"].(float64); code != 0 { + return nil, fmt.Errorf("搜索失败 code=%v", pageJSON["code"]) } - if code, ok := respJSON["code"].(float64); ok && code == 0 { - // 用 UseNumber 重新解析后按 tab_type 过滤: - // 上游 search/tab/v/ 固定返回所有 tab 的 search_tabs; - // 大整数 ID 经 float64 重组会丢精度,必须 UseNumber - dec := json.NewDecoder(bytes.NewReader(resp.Content)) - dec.UseNumber() - var numJSON map[string]interface{} - if err := dec.Decode(&numJSON); err != nil { - return resp.Text, nil // 解析失败时退回原样转发 + filterSearchTabs(pageJSON, tabType) + tabs, _ := pageJSON["search_tabs"].([]interface{}) + for _, t := range tabs { + m, ok := t.(map[string]interface{}) + if !ok { + continue + } + if tt, ok := m["tab_type"].(json.Number); ok { + if v, err := tt.Int64(); err == nil && int(v) == tabType { + return m, nil + } } - filterSearchTabs(numJSON, tabType) - unwrapSearchTab(numJSON) - UnwrapJSONStrings(numJSON) - return ToJSON(numJSON) } + return nil, nil +} - return BuildErrorResponseWithCode(1, "搜索失败"), nil +// idSearchResult 书 ID 直搜：复用详情接口（24h 缓存），包成上游搜索行形状（book_data 包装） +func idSearchResult(bookID string) (string, error) { + detailJSON, err := HandleGetDetail(bookID) + if err != nil { + return BuildErrorResponseWithCode(1, "ID 搜索失败"), err + } + var detail map[string]interface{} + if err := json.Unmarshal([]byte(detailJSON), &detail); err != nil { + return detailJSON, nil // 透传上游错误响应 + } + if code, _ := detail["code"].(float64); code != 0 { + return detailJSON, nil + } + return marshalNoEscape(UnwrapJSONStrings(map[string]interface{}{ + "code": 0, + "message": "SUCCESS", + "data": []interface{}{map[string]interface{}{"book_data": []interface{}{detail["data"]}}}, + "has_more": false, + "next_offset": 0, + })), nil } // filterSearchTabs 按 tab_type 只保留 search_tabs 中匹配的项;无匹配时保留原样 @@ -100,26 +226,44 @@ func filterSearchTabs(respJSON map[string]interface{}, tabType int) { } } -// unwrapSearchTab 将过滤后的唯一 tab 展平：去掉 search_tabs 包装， -// tab 的 data 直接作为顶层 data，并提升分页字段 has_more/next_offset -func unwrapSearchTab(respJSON map[string]interface{}) { - tabs, ok := respJSON["search_tabs"].([]interface{}) - if !ok || len(tabs) != 1 { - return +// searchRowID 取行的 book_id（UseNumber 下为 json.Number，也可能是字符串） +func searchRowID(m map[string]interface{}) string { + if m == nil { + return "" } - tab, ok := tabs[0].(map[string]interface{}) - if !ok { - return + if s, _ := m["book_id"].(string); s != "" { + return s } - delete(respJSON, "search_tabs") - delete(respJSON, "selected_tab_idx") // tab 已展平，索引无意义 - delete(respJSON, "server_ab_params") // 上游 AB 实验参数，透传无意义 - if data, ok := tab["data"]; ok { - respJSON["data"] = data - } - for _, k := range []string{"has_more", "next_offset"} { - if v, ok := tab[k]; ok { - respJSON[k] = v - } + if n, ok := m["book_id"].(json.Number); ok { + return n.String() } + return "" +} + +func searchStr(v interface{}) string { + s, _ := v.(string) + return s +} + +func searchInt(v interface{}) int { + switch t := v.(type) { + case json.Number: + n, _ := t.Int64() + return int(n) + case string: + n, _ := strconv.Atoi(t) + return n + } + return 0 +} + +func searchHasMore(v interface{}) bool { + switch t := v.(type) { + case bool: + return t + case json.Number: + n, _ := t.Int64() + return n != 0 + } + return false } diff --git a/internal/fqapi/shelf.go b/internal/fqapi/shelf.go new file mode 100644 index 0000000..d0a323f --- /dev/null +++ b/internal/fqapi/shelf.go @@ -0,0 +1,194 @@ +package fqapi + +import ( + "fanqie/internal/common" + "fmt" + "strings" +) + +var shelfLogger = common.NewLogger("shelf", "INFO", "") + +// HandleBookshelf 获取登录用户书架。个人动态数据不缓存。 +// group 非空时只返回该分组的书；书架接口对无效/过期 sessionid 也返回空书架，无法区分—— +// 空结果时提示重新登录 +func HandleBookshelf(sessionID, group string) (string, error) { + if sessionID == "" { + return BuildErrorResponse("需要参数 sessionid（在 fanqienovel.com 登录后从浏览器 Cookie 中获取）"), nil + } + + resp, err := signedBookAPIGet("/reading/bookapi/bookshelf/info/v/", nil, sessionID) + if err != nil { + shelfLogger.Error(fmt.Sprintf("获取书架失败: %v", err)) + return BuildErrorResponse("获取书架失败"), err + } + + items, err := bookAPIList(resp, "book_shelf_info") + if err != nil { + return BuildErrorResponse("获取书架失败: " + err.Error()), nil + } + + // 过滤分组并收集 book_id + type shelfRow struct { + id string + groupName string + } + rows := make([]shelfRow, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]interface{}) + if !ok { + continue + } + id := bookIDOf(m) + if id == "" { + continue + } + gn, _ := m["group_name"].(string) + if group != "" && gn != group { + continue + } + rows = append(rows, shelfRow{id: id, groupName: gn}) + } + + if len(rows) == 0 { + return BuildErrorResponse("书架为空：sessionid 可能无效或已过期，请重新获取"), nil + } + + ids := make([]string, len(rows)) + for i, row := range rows { + ids[i] = row.id + } + details := detailMapByIDs(ids) + result := make([]map[string]interface{}, 0, len(rows)) + for _, row := range rows { + d := details[row.id] + if d == nil { + d = map[string]interface{}{"book_id": row.id} + } + d["group_name"] = row.groupName + result = append(result, d) + } + + return marshalNoEscape(UnwrapJSONStrings(buildCacheData(result))), nil +} + +// HandleReadHistory 获取登录用户阅读历史。个人动态数据不缓存。 +// offset 从 0 起，每页 20 条；每项合并书籍详情并保留原始记录（进度等）于 history 字段 +func HandleReadHistory(sessionID string, offset int) (string, error) { + if sessionID == "" { + return BuildErrorResponse("需要参数 sessionid（在 fanqienovel.com 登录后从浏览器 Cookie 中获取）"), nil + } + if offset < 0 { + offset = 0 + } + + resp, err := signedBookAPIGet("/reading/bookapi/read_history/list/v/", + []string{fmt.Sprintf("offset=%d", offset), "book_type=0"}, sessionID) + if err != nil { + shelfLogger.Error(fmt.Sprintf("获取阅读历史失败: %v", err)) + return BuildErrorResponse("获取阅读历史失败"), err + } + + items, err := bookAPIList(resp, "data_list") + if err != nil { + return BuildErrorResponse("获取阅读历史失败: " + err.Error()), nil + } + + type historyRow struct { + id string + raw map[string]interface{} + } + rows := make([]historyRow, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]interface{}) + if !ok { + continue + } + id := bookIDOf(m) + if id == "" { + continue + } + rows = append(rows, historyRow{id: id, raw: m}) + } + + if len(rows) == 0 { + return marshalNoEscape(buildCacheData([]interface{}{})), nil + } + + ids := make([]string, len(rows)) + for i, row := range rows { + ids[i] = row.id + } + details := detailMapByIDs(ids) + result := make([]map[string]interface{}, 0, len(rows)) + for _, row := range rows { + d := details[row.id] + if d == nil { + d = map[string]interface{}{"book_id": row.id} + } + // 原始记录（含阅读进度/时间）挂 history 字段保留 + d["history"] = row.raw + result = append(result, d) + } + + return marshalNoEscape(UnwrapJSONStrings(buildCacheData(result))), nil +} + +// signedBookAPIGet 个人书架/历史接口：常规签名请求 + 登录态 Cookie +func signedBookAPIGet(path string, extraParams []string, sessionID string) (*common.ClientResponse, error) { + reqURL := BuildURL("https://reading.snssdk.com"+path, extraParams...) + return GorgonRequest(reqURL, map[string]interface{}{ + "method": "GET", + "headers": map[string]string{ + "User-Agent": "com.dragon.read", + "Cookie": "sessionid=" + sessionID, + }, + }) +} + +// bookAPIList 校验上游响应并取出 data 下的指定列表字段 +func bookAPIList(resp *common.ClientResponse, field string) ([]interface{}, error) { + respJSON, err := resp.JSON() + if err != nil { + return nil, fmt.Errorf("响应解析失败: %w", err) + } + if code, ok := respJSON["code"].(float64); !ok || code != 0 { + msg, _ := respJSON["message"].(string) + return nil, fmt.Errorf("code=%.0f msg=%s", respJSON["code"], msg) + } + data, _ := respJSON["data"].(map[string]interface{}) + items, _ := data[field].([]interface{}) + return items, nil +} + +// bookIDOf 兼容提取 book_id / book_id_str（数字或字符串） +func bookIDOf(m map[string]interface{}) string { + for _, k := range []string{"book_id_str", "book_id", "bookId"} { + switch v := m[k].(type) { + case string: + if v != "" { + return v + } + case float64: + return fmt.Sprintf("%.0f", v) + } + } + return "" +} + +// detailMapByIDs 按 book_id 建立详情映射（复用批量详情接口，分批请求） +func detailMapByIDs(ids []string) map[string]map[string]interface{} { + m := make(map[string]map[string]interface{}, len(ids)) + for _, chunk := range chunkArray(ids, 100) { + list, err := multiDetail(strings.Join(chunk, ",")) + if err != nil { + shelfLogger.Warning(fmt.Sprintf("批量获取详情失败（该批保留 book_id）: %v", err)) + continue + } + for _, d := range list { + if id, ok := d["book_id"].(string); ok { + m[id] = d + } + } + } + return m +} diff --git a/internal/handlers/book.go b/internal/handlers/book.go index 314ed95..558ec3a 100644 --- a/internal/handlers/book.go +++ b/internal/handlers/book.go @@ -103,3 +103,74 @@ func HandleMultiDetail(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, result) } + +// HandleRank 榜单处理（推荐/完结/热搜/黑马/爆更榜，免签名接口） +func HandleRank(w http.ResponseWriter, r *http.Request) { + incrementStat("total", "category") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + sideType, err := parseIntParam(r.URL.Query().Get("side_type"), "side_type") + if err != nil { + respondError(w, err.Error(), true) + return + } + page, err := parseIntParam(r.URL.Query().Get("page"), "page") + if err != nil { + respondError(w, err.Error(), true) + return + } + + result, err := fqapi.HandleGetRank(sideType, page) + if err != nil { + respondError(w, "获取榜单失败: "+err.Error(), false) + return + } + + fmt.Fprint(w, result) +} + +// HandleBookshelf 书架处理（需登录 sessionid） +func HandleBookshelf(w http.ResponseWriter, r *http.Request) { + incrementStat("total", "category") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + params, err := validateParams(r, "sessionid") + if err != nil { + respondError(w, err.Error(), true) + return + } + + result, err := fqapi.HandleBookshelf(params["sessionid"], r.URL.Query().Get("group")) + if err != nil { + respondError(w, "获取书架失败: "+err.Error(), false) + return + } + + fmt.Fprint(w, result) +} + +// HandleReadHistory 阅读历史处理（需登录 sessionid） +func HandleReadHistory(w http.ResponseWriter, r *http.Request) { + incrementStat("total", "category") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + params, err := validateParams(r, "sessionid") + if err != nil { + respondError(w, err.Error(), true) + return + } + + offset, err := parseIntParam(r.URL.Query().Get("offset"), "offset") + if err != nil { + respondError(w, err.Error(), true) + return + } + + result, err := fqapi.HandleReadHistory(params["sessionid"], offset) + if err != nil { + respondError(w, "获取阅读历史失败: "+err.Error(), false) + return + } + + fmt.Fprint(w, result) +} diff --git a/internal/handlers/category.go b/internal/handlers/category.go index 462b18f..a5784bc 100644 --- a/internal/handlers/category.go +++ b/internal/handlers/category.go @@ -32,8 +32,10 @@ func HandleCategoryLanding(w http.ResponseWriter, r *http.Request) { wordNumber := r.URL.Query().Get("word_number") bookStatus := r.URL.Query().Get("book_status") sortBy := r.URL.Query().Get("sort_by") + // 新版分类标签自带的筛选项（见 /category-front 返回的 category_landpage_url 参数） + selectedItems := r.URL.Query().Get("selected_items") - result, err := fqapi.HandleCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy) + result, err := fqapi.HandleCategoryLanding(categoryID, offset, wordNumber, bookStatus, sortBy, selectedItems) if err != nil { respondError(w, "获取分类详情失败: "+err.Error(), false) return diff --git a/internal/handlers/common.go b/internal/handlers/common.go index 6dc8da2..65be575 100644 --- a/internal/handlers/common.go +++ b/internal/handlers/common.go @@ -270,7 +270,8 @@ func LogRequestMiddleware(handler http.HandlerFunc) http.HandlerFunc { } } -// decodedQuery 重组解码后的查询串（keyword=%E6%B5%8B → keyword=测），日志可读 +// decodedQuery 重组解码后的查询串（keyword=%E6%B5%8B → keyword=测），日志可读； +// key 参数值脱敏（访问密钥不落日志） // ponytail: map 遍历乱序、%26 解码后与分隔符同名 —— 展示用途可接受 func decodedQuery(r *http.Request) string { vals := r.URL.Query() // 已解码 @@ -280,6 +281,9 @@ func decodedQuery(r *http.Request) string { parts := make([]string, 0, len(vals)) for k, vs := range vals { for _, v := range vs { + if strings.EqualFold(k, "key") && v != "" { + v = "***" + } parts = append(parts, k+"="+v) } } diff --git a/openapi.yaml b/openapi.yaml index 7353155..a99147c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -710,6 +710,10 @@ paths: in: query description: 排序方式,透传上游(0=默认) schema: { type: integer, default: 0 } + - name: selected_items + in: query + description: 新版分类标签自带筛选项(从 /category-front 返回的 category_landpage_url 参数中获取),透传上游 + schema: { type: string } responses: "200": description: 分类书籍列表(上游转发) @@ -721,6 +725,85 @@ paths: $ref: "#/components/responses/BadRequest" "500": $ref: "#/components/responses/ServerError" + /rank: + get: + tags: [榜单与个人数据] + summary: 获取书籍榜单 + description: 免签名接口,支持五种榜单,每页 20 本,短缓存 10 分钟。 + operationId: getRank + parameters: + - name: side_type + in: query + description: 榜单类型 + schema: + type: integer + default: 10 + enum: [10, 11, 12, 13, 15] + x-enum-varnames: [推荐榜, 完结榜, 热搜榜, 黑马榜, 爆更榜] + - name: page + in: query + description: 页码,从 1 起 + schema: { type: integer, default: 1, minimum: 1 } + responses: + "200": + description: 榜单书籍列表(上游转发,data.result 为书籍数组) + content: + application/json: + schema: { $ref: "#/components/schemas/UpstreamResponse" } + "500": + $ref: "#/components/responses/ServerError" + /bookshelf: + get: + tags: [榜单与个人数据] + summary: 获取我的书架 + description: 需要番茄网页版登录态;每本书合并详情数据并附带 group_name。个人动态数据不缓存。 + operationId: getBookshelf + parameters: + - name: sessionid + in: query + required: true + description: 番茄网页版(fanqienovel.com)登录后的 Cookie sessionid + schema: { type: string } + - name: group + in: query + description: 按分组过滤,为空返回全部 + schema: { type: string } + responses: + "200": + description: 书架书籍列表(data 为数组,每项含书籍详情 + group_name) + content: + application/json: + schema: { $ref: "#/components/schemas/UpstreamResponse" } + "400": + $ref: "#/components/responses/BadRequest" + "500": + $ref: "#/components/responses/ServerError" + /read-history: + get: + tags: [榜单与个人数据] + summary: 获取阅读历史 + description: 需要番茄网页版登录态;每页 20 条,每项合并书籍详情并保留原始记录(含阅读进度)于 history 字段。个人动态数据不缓存。 + operationId: getReadHistory + parameters: + - name: sessionid + in: query + required: true + description: 番茄网页版(fanqienovel.com)登录后的 Cookie sessionid + schema: { type: string } + - name: offset + in: query + description: 偏移量,默认 0,每页 20 条 + schema: { type: integer, default: 0, minimum: 0 } + responses: + "200": + description: 阅读历史列表(data 为数组) + content: + application/json: + schema: { $ref: "#/components/schemas/UpstreamResponse" } + "400": + $ref: "#/components/responses/BadRequest" + "500": + $ref: "#/components/responses/ServerError" /emoji/{file}: get: tags: [静态资源]