Compare commits

..
4 Commits
18 changed files with 756 additions and 13 deletions
+6
View File
@@ -37,6 +37,12 @@ func NewRouter(cfg *config.Config, st *store.Store, rdb *redispkg.R, sc *scanner
libs.POST("", h.AdminOnly(), h.CreateLibrary) libs.POST("", h.AdminOnly(), h.CreateLibrary)
libs.POST("/:id/scan", h.AdminOnly(), h.ScanLibrary) libs.POST("/:id/scan", h.AdminOnly(), h.ScanLibrary)
libs.POST("/:id/upload", h.AdminOnly(), h.Upload) libs.POST("/:id/upload", h.AdminOnly(), h.Upload)
libs.POST("/:id/upload/init", h.AdminOnly(), h.UploadInit)
uploads := p.Group("/uploads", h.AdminOnly())
uploads.GET("/:uid", h.UploadStatus)
uploads.POST("/:uid/complete", h.UploadComplete)
uploads.PUT("/:uid/parts/:index", h.UploadPart)
p.GET("/books", h.ListBooks) p.GET("/books", h.ListBooks)
p.GET("/books/:id", h.GetBook) p.GET("/books/:id", h.GetBook)
+1 -1
View File
@@ -24,7 +24,7 @@ import (
) )
func testCfg() *config.Config { func testCfg() *config.Config {
return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 200, return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 1, // 测试里 1MB:超限用例只需 2MB body
TrustedProxies: []string{"172.16.0.0/12"}} // 与 prod 默认一致: 只有 compose 网段内代理才可信 TrustedProxies: []string{"172.16.0.0/12"}} // 与 prod 默认一致: 只有 compose 网段内代理才可信
} }
+44 -2
View File
@@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
@@ -118,7 +119,7 @@ func (h *H) openBook(c *gin.Context, b store.Book, root string) (*os.File, int64
func (h *H) pageIndex(c *gin.Context, b store.Book, root string) ([]string, error) { func (h *H) pageIndex(c *gin.Context, b store.Book, root string) ([]string, error) {
hash := bookfile.Hash(b.FileSize, b.ModTS) hash := bookfile.Hash(b.FileSize, b.ModTS)
key := fmt.Sprintf("pagesidx:%d:%s", b.ID, hash) key := fmt.Sprintf("pagesidx2:%d:%s", b.ID, hash) // 前缀换代 = 索引逻辑变更时一次性作废旧缓存
if v, ok := h.rdb.Get(c, key); ok && v != "" { if v, ok := h.rdb.Get(c, key); ok && v != "" {
return strings.Split(v, "\n"), nil return strings.Split(v, "\n"), nil
} }
@@ -158,7 +159,48 @@ func (h *H) PagesCount(c *gin.Context) {
err(c, http.StatusUnprocessableEntity, "broken", e.Error()) err(c, http.StatusUnprocessableEntity, "broken", e.Error())
return return
} }
c.JSON(http.StatusOK, gin.H{"count": len(idx)}) c.JSON(http.StatusOK, gin.H{"count": len(idx), "chapters": chaptersOf(idx)})
}
type cbzChapter struct {
Title string `json:"title"`
Start int `json:"start"`
}
// chaptersOf 页索引已自然序排列,按父目录分组:每个新目录开一章,标题取目录名;扁平包或只有一组返回 nil
func chaptersOf(idx []string) []cbzChapter {
type grp struct {
dir string
start int
}
var grps []grp
last := "\x00"
for i, n := range idx {
d := path.Dir(n)
if d == last {
continue
}
last = d
if d == "." { // 根目录散页不开章
continue
}
grps = append(grps, grp{d, i})
}
if len(grps) < 2 {
return nil
}
titles := make(map[string]int)
out := make([]cbzChapter, len(grps))
for i, g := range grps {
out[i] = cbzChapter{Title: path.Base(g.dir), Start: g.start}
titles[out[i].Title]++
}
for i, g := range grps { // 同名目录(不同父级)撞车 → 用全路径消歧
if titles[out[i].Title] > 1 {
out[i].Title = g.dir
}
}
return out
} }
func (h *H) Page(c *gin.Context) { func (h *H) Page(c *gin.Context) {
@@ -1,6 +1,8 @@
package handlers_test package handlers_test
import ( import (
"archive/zip"
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"net/http" "net/http"
@@ -103,6 +105,76 @@ func TestPages(t *testing.T) {
} }
} }
func zipTo(t *testing.T, path string, kv map[string]string) {
t.Helper()
os.MkdirAll(filepath.Dir(path), 0o755)
buf := &bytes.Buffer{}
zw := zip.NewWriter(buf)
for n, v := range kv {
w, err := zw.Create(n)
if err != nil {
t.Fatal(err)
}
w.Write([]byte(v))
}
zw.Close()
os.WriteFile(path, buf.Bytes(), 0o644)
}
func TestPagesChaptersAndNoImageZip(t *testing.T) {
st, sc, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
lib, root := newLibrary(t, st, h, tok, booksDir, "ch")
zipTo(t, filepath.Join(root, "show.cbz"), map[string]string{
"第2季/第2話/0001.jpg": "IMG2",
"第2季/第1話/0001.jpg": "IMG1a",
"第2季/第1話/._0001.jpg": "junk",
"__MACOSX/第2季/._0001.jpg": "junk",
})
zipTo(t, filepath.Join(root, "videos.zip"), map[string]string{"ep/01.mkv": "x"})
scanNow(t, sc, lib)
bookID := func(q string) string {
w := do(h, "GET", "/api/books?q="+q, tok, nil)
var bs []map[string]any
json.Unmarshal(w.Body.Bytes(), &bs)
if len(bs) != 1 {
t.Fatalf("q=%s books: %s", q, w.Body)
}
return itoa(bs[0]["id"])
}
cbzID := bookID("show")
w := do(h, "GET", "/api/books/"+cbzID+"/pages", tok, nil)
var pg struct {
Count int `json:"count"`
Chapters []struct {
Title string `json:"title"`
Start int `json:"start"`
} `json:"chapters"`
}
json.Unmarshal(w.Body.Bytes(), &pg)
if pg.Count != 2 || len(pg.Chapters) != 2 {
t.Fatalf("pages want 2/2ch got %s", w.Body)
}
if pg.Chapters[0].Title != "第1話" || pg.Chapters[0].Start != 0 || pg.Chapters[1].Start != 1 {
t.Fatalf("chapters: %s", w.Body)
}
ww := do(h, "GET", "/api/books/"+cbzID+"/pages/0", tok, nil)
if ww.Code != 200 || ww.Body.String() != "IMG1a" {
t.Fatalf("page0 must be real image (junk filtered): %d %s", ww.Code, ww.Body)
}
if ww := do(h, "GET", "/api/books/"+cbzID+"/pages/2", tok, nil); ww.Code != 404 {
t.Fatalf("junk-indexed page must be gone: %d", ww.Code)
}
// 无图 zip → state=error,而不是空白 reader
w = do(h, "GET", "/api/books/"+bookID("videos"), tok, nil)
var bk map[string]any
json.Unmarshal(w.Body.Bytes(), &bk)
if bk["state"] != "error" || !strings.Contains(bk["error"].(string), "no images") {
t.Fatalf("empty-image zip want error, got %s", w.Body)
}
}
func TestBrokenCBZ(t *testing.T) { func TestBrokenCBZ(t *testing.T) {
st, sc, h, booksDir := setupAPI(t) st, sc, h, booksDir := setupAPI(t)
atok := adminToken(t, h) atok := adminToken(t, h)
+5
View File
@@ -116,6 +116,11 @@ func (h *H) Upload(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.cfg.UploadMaxMB<<20) c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.cfg.UploadMaxMB<<20)
fh, e := c.FormFile("file") fh, e := c.FormFile("file")
if e != nil { if e != nil {
var mbe *http.MaxBytesError
if errors.As(e, &mbe) {
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
return
}
err(c, http.StatusBadRequest, "bad_request", "multipart field 'file' required") err(c, http.StatusBadRequest, "bad_request", "multipart field 'file' required")
return return
} }
@@ -80,6 +80,41 @@ func TestLibraryCreateListUpload(t *testing.T) {
} }
} }
func TestUploadSizeAndFilename(t *testing.T) {
_, _, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": "s2"})
if w.Code != 201 {
t.Fatalf("create lib %d %s", w.Code, w.Body)
}
var lib map[string]any
json.Unmarshal(w.Body.Bytes(), &lib)
libID := itoa(lib["id"])
// 全角冒号等非 ASCII 文件名正常落盘
body, mw := uploadBody("調教開關:第二季.cbz", []byte("zipbytes"))
req := httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+tok)
ww := httptest.NewRecorder()
h.ServeHTTP(ww, req)
if ww.Code != 202 {
t.Fatalf("cjk name upload %d %s", ww.Code, ww.Body)
}
if _, err := os.Stat(filepath.Join(booksDir, "s2", "調教開關:第二季.cbz")); err != nil {
t.Fatal("cjk upload missing:", err)
}
// 超过 UploadMaxMB(测试=1MB)→ 413 too_large,而非误报 "file required"
body, mw = uploadBody("big.cbz", bytes.Repeat([]byte("x"), 2<<20))
req = httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+tok)
ww = httptest.NewRecorder()
h.ServeHTTP(ww, req)
if ww.Code != 413 || !strings.Contains(ww.Body.String(), "too_large") {
t.Fatalf("oversize want 413 too_large got %d %s", ww.Code, ww.Body)
}
}
func uploadBody(filename string, content []byte) (*bytes.Buffer, *multipart.Writer) { func uploadBody(filename string, content []byte) (*bytes.Buffer, *multipart.Writer) {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
mw := multipart.NewWriter(buf) mw := multipart.NewWriter(buf)
+285
View File
@@ -0,0 +1,285 @@
package handlers
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"booklib/internal/bookfile"
)
// 分片上传:init(指纹→确定性 uploadId,天然支持续传)→ PUT parts → complete 拼接原子落盘。
// 会话即 <BooksDir>/.uploads/<uid>/(meta.json + parts/N),无独立状态存储;24h 未 complete opportunistic 清扫。
const (
maxChunkBytes = 32 << 20
defaultChunk = 8 << 20
uploadSessTTL = 24 * time.Hour
uploadSessionIn = ".uploads"
)
type uploadMeta struct {
Name string `json:"name"`
Size int64 `json:"size"`
ChunkSize int64 `json:"chunkSize"`
LibraryID int64 `json:"libraryId"`
}
func validUploadID(s string) bool {
if len(s) != 32 {
return false
}
for _, r := range s {
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) {
return false
}
}
return true
}
func uploadIDFor(libID int64, name string, size, chunk int64) string {
h := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%d|%d", libID, name, size, chunk)))
return hex.EncodeToString(h[:16])
}
func (h *H) uploadDir(uid string) string {
return filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn, uid)
}
func chunkRange(m uploadMeta, i int64) (int64, int64) {
lo := i * m.ChunkSize
hi := min(lo+m.ChunkSize, m.Size)
return lo, hi
}
func (h *H) numParts(m uploadMeta) int64 {
return (m.Size + m.ChunkSize - 1) / m.ChunkSize
}
// sweepUploads 删除过期会话目录;尽力而为,失败不影响主流程
func (h *H) sweepUploads() {
base := filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn)
es, e := os.ReadDir(base)
if e != nil {
return
}
for _, en := range es {
if fi, e := en.Info(); e == nil && time.Since(fi.ModTime()) > uploadSessTTL {
os.RemoveAll(filepath.Join(base, en.Name()))
}
}
}
func (h *H) UploadInit(c *gin.Context) {
lib, ok := h.getLibrary(c)
if !ok {
return
}
if _, ok := h.libRoot(c, lib); !ok {
return
}
var req struct {
Name string `json:"name"`
Size int64 `json:"size"`
ChunkSize int64 `json:"chunkSize"`
}
if c.ShouldBindJSON(&req) != nil {
err(c, http.StatusBadRequest, "bad_request", "name, size required")
return
}
name := bookfile.SafeName(req.Name)
if name == "" {
err(c, http.StatusBadRequest, "bad_request", "bad name")
return
}
if bookfile.FormatFromExt(name) == "" {
err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md")
return
}
if req.Size <= 0 {
err(c, http.StatusBadRequest, "bad_request", "bad size")
return
}
if req.Size > h.cfg.UploadMaxMB<<20 {
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
return
}
if req.ChunkSize == 0 {
req.ChunkSize = defaultChunk
}
if req.ChunkSize > maxChunkBytes {
err(c, http.StatusBadRequest, "bad_request", "chunkSize must be <= 33554432")
return
}
uid := uploadIDFor(lib.ID, name, req.Size, req.ChunkSize)
dir := h.uploadDir(uid)
meta := uploadMeta{Name: name, Size: req.Size, ChunkSize: req.ChunkSize, LibraryID: lib.ID}
if b, e := os.ReadFile(filepath.Join(dir, "meta.json")); e == nil {
var old uploadMeta
if json.Unmarshal(b, &old) == nil && old == meta { // 同指纹 → 复用会话(续传)
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
return
}
os.RemoveAll(dir) // 指纹撞上但内容不同 → 从头再来
}
h.sweepUploads()
if e := os.MkdirAll(filepath.Join(dir, "parts"), 0o755); e != nil {
err(c, http.StatusInternalServerError, "internal", "create session")
return
}
b, _ := json.Marshal(meta)
if e := os.WriteFile(filepath.Join(dir, "meta.json"), b, 0o644); e != nil {
err(c, http.StatusInternalServerError, "internal", "write meta")
return
}
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
}
// loadMeta 校验 uid 与路径,404/400 已回复
func (h *H) loadMeta(c *gin.Context) (uploadMeta, string, bool) {
uid := c.Param("uid")
if !validUploadID(uid) {
err(c, http.StatusBadRequest, "bad_request", "bad upload id")
return uploadMeta{}, "", false
}
dir := h.uploadDir(uid)
var m uploadMeta
b, e := os.ReadFile(filepath.Join(dir, "meta.json"))
if e != nil {
err(c, http.StatusNotFound, "not_found", "no such upload")
return uploadMeta{}, "", false
}
if json.Unmarshal(b, &m) != nil {
err(c, http.StatusInternalServerError, "internal", "corrupt session")
return uploadMeta{}, "", false
}
return m, dir, true
}
func (h *H) UploadStatus(c *gin.Context) {
_, dir, ok := h.loadMeta(c)
if !ok {
return
}
recv := []int64{}
es, e := os.ReadDir(filepath.Join(dir, "parts"))
if e == nil {
for _, en := range es {
if i, e := strconv.ParseInt(en.Name(), 10, 64); e == nil {
recv = append(recv, i)
}
}
}
sort.Slice(recv, func(i, j int) bool { return recv[i] < recv[j] })
c.JSON(http.StatusOK, gin.H{"received": recv})
}
func (h *H) UploadPart(c *gin.Context) {
m, dir, ok := h.loadMeta(c)
if !ok {
return
}
idx, e := strconv.ParseInt(c.Param("index"), 10, 64)
if e != nil || idx < 0 || idx >= h.numParts(m) {
err(c, http.StatusBadRequest, "bad_request", "bad part index")
return
}
lo, hi := chunkRange(m, idx)
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, hi-lo)
p := filepath.Join(dir, "parts", strconv.FormatInt(idx, 10))
f, e := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if e != nil {
err(c, http.StatusInternalServerError, "internal", "create part")
return
}
n, e := io.Copy(f, c.Request.Body)
f.Close()
if e != nil || n != hi-lo {
os.Remove(p)
var mbe *http.MaxBytesError
code, msg := "too_large", "part size mismatch"
if errors.As(e, &mbe) {
msg = "part exceeds declared size"
}
err(c, http.StatusRequestEntityTooLarge, code, msg)
return
}
c.JSON(http.StatusAccepted, gin.H{"accepted": true})
}
func (h *H) UploadComplete(c *gin.Context) {
m, dir, ok := h.loadMeta(c)
if !ok {
return
}
var total int64
for i := int64(0); i < h.numParts(m); i++ {
lo, hi := chunkRange(m, i)
fi, e := os.Stat(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
if e != nil || fi.Size() != hi-lo {
err(c, http.StatusBadRequest, "bad_request", "upload incomplete; missing or corrupt parts, re-upload them")
return
}
total += fi.Size()
}
if total != m.Size {
err(c, http.StatusBadRequest, "bad_request", "total size mismatch")
return
}
lib, e := h.st.GetLibrary(c, m.LibraryID)
if e != nil {
dbErr(c, e)
return
}
root, ok := h.libRoot(c, lib)
if !ok {
return
}
dst, e := h.uniquePath(root, m.Name)
if e != nil {
err(c, http.StatusForbidden, "forbidden", e.Error())
return
}
tmp := filepath.Join(dir, "assembled")
out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if e != nil {
err(c, http.StatusInternalServerError, "internal", "create tmp")
return
}
for i := int64(0); i < h.numParts(m); i++ {
pf, e := os.Open(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
if e != nil {
out.Close()
err(c, http.StatusInternalServerError, "internal", "open part")
return
}
if _, e := io.Copy(out, pf); e != nil {
pf.Close()
out.Close()
os.Remove(tmp)
err(c, http.StatusInternalServerError, "internal", "assemble")
return
}
pf.Close()
}
out.Close()
if e := os.Rename(tmp, dst); e != nil { // 原子落盘,scanner 自动收编
os.Remove(tmp)
err(c, http.StatusInternalServerError, "internal", "rename")
return
}
os.RemoveAll(dir)
c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": strings.TrimPrefix(dst, root+string(os.PathSeparator))})
}
+157
View File
@@ -0,0 +1,157 @@
package handlers_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
func initUpload(t *testing.T, h http.Handler, tok, libID, name string, size, chunk int64) (map[string]any, *httptest.ResponseRecorder) {
t.Helper()
w := do(h, "POST", "/api/libraries/"+libID+"/upload/init", tok,
map[string]any{"name": name, "size": size, "chunkSize": chunk})
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return m, w
}
func putPart(h http.Handler, tok, uid string, idx int, data []byte) *httptest.ResponseRecorder {
req := httptest.NewRequest("PUT", "/api/uploads/"+uid+"/parts/"+fmt.Sprint(idx), bytes.NewReader(data))
req.Header.Set("Authorization", "Bearer "+tok)
req.Header.Set("Content-Type", "application/octet-stream")
ww := httptest.NewRecorder()
h.ServeHTTP(ww, req)
return ww
}
func mkLib(t *testing.T, h http.Handler, tok, name string) string {
t.Helper()
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": name})
if w.Code != 201 {
t.Fatalf("create lib %d %s", w.Code, w.Body)
}
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return itoa(m["id"])
}
func TestUploadChunkedHappyPath(t *testing.T) {
_, _, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("調教開關第二季!"), 40000) // ~880KB, <1MB 测试上限
size := int64(len(content))
chunk := int64(400000)
m, w := initUpload(t, h, tok, libID, "調教開關:第二季.zip", size, chunk)
if w.Code != 200 || m["uploadId"] == "" {
t.Fatalf("init %d %s", w.Code, w.Body)
}
uid := m["uploadId"].(string)
n := int((size + chunk - 1) / chunk) // 乱序上传
for _, i := range []int{2, 0, 1} {
lo, hi := int64(i)*chunk, int64(i+1)*chunk
if hi > size {
hi = size
}
if ww := putPart(h, tok, uid, i, content[lo:hi]); ww.Code != 202 {
t.Fatalf("part %d: %d %s", i, ww.Code, ww.Body)
}
}
w = do(h, "GET", "/api/uploads/"+uid, tok, nil)
var st struct{ Received []int }
json.Unmarshal(w.Body.Bytes(), &st)
if len(st.Received) != n {
t.Fatalf("status want %d got %v", n, st.Received)
}
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 202 {
t.Fatalf("complete %d %s", w.Code, w.Body)
}
var res map[string]any
json.Unmarshal(w.Body.Bytes(), &res)
if res["path"] != "調教開關:第二季.zip" {
t.Fatalf("path: %v", res["path"])
}
got, err := os.ReadFile(filepath.Join(booksDir, "s2", "調教開關:第二季.zip"))
if err != nil || !bytes.Equal(got, content) {
t.Fatalf("assembled file wrong: err=%v eq=%v", err, bytes.Equal(got, content))
}
if entries, _ := os.ReadDir(filepath.Join(booksDir, ".uploads")); len(entries) != 0 {
t.Fatalf("session not cleaned: %v", entries)
}
}
func TestUploadChunkedResumeKeepsParts(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("x"), 900000)
m, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
uid := m["uploadId"].(string)
putPart(h, tok, uid, 0, content[:400000])
// 同指纹重复 init → 复用会话,已传分片保留
m2, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
if m2["uploadId"] != uid {
t.Fatalf("resume want same uid got %v", m2["uploadId"])
}
w := do(h, "GET", "/api/uploads/"+uid, tok, nil)
if !strings.Contains(w.Body.String(), "[0]") {
t.Fatalf("resumed status: %s", w.Body)
}
}
func TestUploadChunkedErrors(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
// 总量超 UPLOAD_MAX_MB(测试=1MB)→ 413,消息带限额
_, w := initUpload(t, h, tok, libID, "big.cbz", 2<<20, 400000)
if w.Code != 413 || !strings.Contains(w.Body.String(), "too_large") {
t.Fatalf("oversize want 413 got %d %s", w.Code, w.Body)
}
// 扩展名白名单
_, w = initUpload(t, h, tok, libID, "virus.exe", 100, 10)
if w.Code != 400 || !strings.Contains(w.Body.String(), "bad_format") {
t.Fatalf("bad ext want 400 got %d %s", w.Code, w.Body)
}
// chunkSize 超 32MB
_, w = initUpload(t, h, tok, libID, "ok.cbz", 100, 40<<20)
if w.Code != 400 {
t.Fatalf("huge chunk want 400 got %d %s", w.Code, w.Body)
}
m, _ := initUpload(t, h, tok, libID, "p.cbz", 1000, 400)
uid := m["uploadId"].(string)
// 越界 index
if ww := putPart(h, tok, uid, 9, bytes.Repeat([]byte("y"), 400)); ww.Code != 400 {
t.Fatalf("index oob want 400 got %d %s", ww.Code, ww.Body)
}
// 分片超期望体积
if ww := putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 500)); ww.Code != 413 {
t.Fatalf("part too big want 413 got %d %s", ww.Code, ww.Body)
}
// 缺片 complete
putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 400))
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 400 {
t.Fatalf("missing parts want 400 got %d %s", w.Code, w.Body)
}
// 未知 uid → 404
w = do(h, "GET", "/api/uploads/deadbeefdeadbeefdeadbeefdeadbeef", tok, nil)
if w.Code != 404 {
t.Fatalf("unknown uid want 404 got %d", w.Code)
}
// 非法 uid → 400
w = do(h, "GET", "/api/uploads/zzz", tok, nil)
if w.Code != 400 {
t.Fatalf("bad uid want 400 got %d %s", w.Code, w.Body)
}
}
+11
View File
@@ -26,6 +26,14 @@ func isImage(name string) bool {
return false return false
} }
// junkEntry: macOS 打包混入的资源叉垃圾(__MACOSX/ 目录与 ._* AppleDouble),不是页
func junkEntry(name string) bool {
if strings.HasPrefix(strings.ToLower(name), "__macosx/") {
return true
}
return strings.HasPrefix(path.Base(name), "._")
}
func PageIndex(f io.ReaderAt, size int64) ([]string, error) { func PageIndex(f io.ReaderAt, size int64) ([]string, error) {
zr, err := zip.NewReader(f, size) zr, err := zip.NewReader(f, size)
if err != nil { if err != nil {
@@ -36,6 +44,9 @@ func PageIndex(f io.ReaderAt, size int64) ([]string, error) {
if unsafeEntry(zf.Name) { if unsafeEntry(zf.Name) {
return nil, fmt.Errorf("%w: %s", ErrUnsafeZip, zf.Name) return nil, fmt.Errorf("%w: %s", ErrUnsafeZip, zf.Name)
} }
if junkEntry(zf.Name) {
continue
}
if isImage(zf.Name) { if isImage(zf.Name) {
names = append(names, zf.Name) names = append(names, zf.Name)
} }
+17
View File
@@ -38,6 +38,23 @@ func TestPageIndexSortAndFilter(t *testing.T) {
} }
} }
func TestPageIndexSkipsAppleDouble(t *testing.T) {
r := zipOf(t, "第2季/第2話/0001.jpg", "第2季/第1話/._0001.jpg", "__MACOSX/第2季/._0001.jpg", "第2季/第1話/0001.jpg", "._top.jpg")
idx, err := PageIndex(r, int64(r.Len()))
if err != nil {
t.Fatal(err)
}
want := []string{"第2季/第1話/0001.jpg", "第2季/第2話/0001.jpg"}
if len(idx) != len(want) {
t.Fatalf("got %v want %v", idx, want)
}
for i := range want {
if idx[i] != want[i] {
t.Fatalf("got %v want %v", idx, want)
}
}
}
func TestPageIndexRejectsSlip(t *testing.T) { func TestPageIndexRejectsSlip(t *testing.T) {
for _, bad := range []string{"../evil.jpg", "/etc/passwd.jpg", "a\\..\\b.jpg", "pag\ne.jpg", "pag\re.jpg"} { for _, bad := range []string{"../evil.jpg", "/etc/passwd.jpg", "a\\..\\b.jpg", "pag\ne.jpg", "pag\re.jpg"} {
r := zipOf(t, bad) r := zipOf(t, bad)
+7
View File
@@ -2,6 +2,7 @@ package scanner
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
@@ -146,6 +147,9 @@ func (s *Scanner) add(ctx context.Context, libID int64, root, rel string, ds dis
idx, err := s.zipIndex(root, rel) idx, err := s.zipIndex(root, rel)
pageCount = len(idx) pageCount = len(idx)
idxErr = err idxErr = err
if idxErr == nil && pageCount == 0 { // 视频/文档 zip 不是漫画,空白 reader 没有意义
idxErr = errors.New("no images in archive")
}
} }
id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount) id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount)
if err != nil { if err != nil {
@@ -167,6 +171,9 @@ func (s *Scanner) update(ctx context.Context, libID, bookID int64, root, rel str
idx, err := s.zipIndex(root, rel) idx, err := s.zipIndex(root, rel)
pageCount = len(idx) pageCount = len(idx)
idxErr = err idxErr = err
if idxErr == nil && pageCount == 0 { // 视频/文档 zip 不是漫画,空白 reader 没有意义
idxErr = errors.New("no images in archive")
}
} }
if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil { if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil {
log.Printf("scan: update %s: %v", rel, err) log.Printf("scan: update %s: %v", rel, err)
+3 -1
View File
@@ -2,10 +2,12 @@ JWT_SECRET=change-me-openssl-rand-hex-32
ADMIN_USER=admin ADMIN_USER=admin
ADMIN_PASSWORD=change-me-min-8 ADMIN_PASSWORD=change-me-min-8
SCAN_INTERVAL_SEC=60 SCAN_INTERVAL_SEC=60
# 上传总大小上限(MB)。前端 >16MB 自动分片(每片 8MB),nginx 体积限制只需盖住单个分片
UPLOAD_MAX_MB=2048
# ---- 部署参数(deploy/prepare.sh 渲染模板 / compose 插值用) ---- # ---- 部署参数(deploy/prepare.sh 渲染模板 / compose 插值用) ----
WEB_PORT=8080 WEB_PORT=8080
NGINX_CLIENT_MAX_BODY_SIZE=200m NGINX_CLIENT_MAX_BODY_SIZE=32m
REDIS_MAXMEMORY=128mb REDIS_MAXMEMORY=128mb
REDIS_MAXMEMORY_POLICY=allkeys-lru REDIS_MAXMEMORY_POLICY=allkeys-lru
DELVE_PORT=2345 DELVE_PORT=2345
+1
View File
@@ -28,6 +28,7 @@ services:
BOOKS_DIR: /data/books BOOKS_DIR: /data/books
CACHE_DIR: /data/books/cache CACHE_DIR: /data/books/cache
SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60} SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60}
UPLOAD_MAX_MB: ${UPLOAD_MAX_MB:-200}
volumes: volumes:
- ../backend:/app - ../backend:/app
- ./api/storage:/data/books - ./api/storage:/data/books
+1
View File
@@ -22,6 +22,7 @@ services:
BOOKS_DIR: /data/books BOOKS_DIR: /data/books
CACHE_DIR: /data/books/cache CACHE_DIR: /data/books/cache
SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60} SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60}
UPLOAD_MAX_MB: ${UPLOAD_MAX_MB:-200}
volumes: volumes:
- ./api/storage:/data/books - ./api/storage:/data/books
depends_on: depends_on:
+13
View File
@@ -8,8 +8,21 @@ The format loosely follows Keep a Changelog and can be adapted to the team's hab
## [Unreleased] ## [Unreleased]
### Added / 新增
- API: CBZ page indexing now skips macOS packaging junk (`__MACOSX/…` and `._*` AppleDouble files), which used to land in the page list as ~163-byte black "pages"; `GET /api/books/:id/pages` additionally returns `chapters:[{title,start}]` derived from the archive's folder structure (e.g. 第1話…), so per-folder comics expose their real organization.
- API:CBZ 页索引现会跳过 macOS 打包垃圾(`__MACOSX/…` 与 `._*` 资源叉文件),此前它们以 ~163 字节黑页混入页列表;`GET /api/books/:id/pages` 新增 `chapters:[{title,start}]`,按压缩包内目录结构(如 第1話…)给出真实章节。
- API: resumable chunked upload protocol for large files — `POST /api/libraries/:id/upload/init` (fingerprint-derived deterministic `uploadId`, rejects totals over `UPLOAD_MAX_MB` with `413 too_large`), `PUT /api/uploads/:uid/parts/:index` (parts ≤ 32MB), `GET /api/uploads/:uid` (received parts, for resume), `POST /api/uploads/:uid/complete` (assemble + atomic land, same path contract as single-POST upload). Sessions persist under `BOOKS_DIR/.uploads/` with 24h opportunistic sweep. `UPLOAD_MAX_MB` is now wired through both compose stacks/`.env`; `.env.example` sets 2048 and drops `NGINX_CLIENT_MAX_BODY_SIZE` to 32m (nginx only ever sees one chunk).
- API: 新增大文件可续传分片上传协议——`POST /api/libraries/:id/upload/init`(按指纹派生确定性 `uploadId`,总量超 `UPLOAD_MAX_MB` 返回 `413 too_large`)、`PUT /api/uploads/:uid/parts/:index`(单片 ≤32MB)、`GET /api/uploads/:uid`(查询已传分片以续传)、`POST /api/uploads/:uid/complete`(拼接后原子落盘,返回与单发上传一致的 `path`)。会话存于 `BOOKS_DIR/.uploads/`,超 24h 顺手清理。`UPLOAD_MAX_MB` 已接入两份 compose/`.env`;`.env.example` 调至 2048 并将 `NGINX_CLIENT_MAX_BODY_SIZE` 降为 32m(nginx 只见单个分片)。
### Changed / 变更 ### Changed / 变更
- Scanner: an image-list-less archive (`.zip`/`.cbz` with no page images — video packs, document dumps) is now recorded as `state=error` ("no images in archive") instead of registering as an empty CBZ with a blank reader.
- 扫描器:不含任何图片条目的 `.zip`/`.cbz`(视频包、文档包)现记录为 `state=error`("no images in archive"),不再注册成空 CBZ 留下一个白板阅读器。
- API: upload over `UPLOAD_MAX_MB` now returns `413 too_large` with the limit in the message; previously the size abort was misreported as `400 bad_request "multipart field 'file' required"`.
- API:超过 `UPLOAD_MAX_MB` 的上传现在返回 `413 too_large` 并在消息中带上限额;此前体积超限被误报为 `400 bad_request "multipart field 'file' required"`。
- API: `POST /api/libraries` now takes only `{name}`; `root_path` is generated server-side as `BOOKS_DIR/<sanitized name>` (no client-supplied paths, validated at creation). - API: `POST /api/libraries` now takes only `{name}`; `root_path` is generated server-side as `BOOKS_DIR/<sanitized name>` (no client-supplied paths, validated at creation).
- API:`POST /api/libraries` 只需 `{name}`;`root_path` 由服务端生成为 `BOOKS_DIR/<清洗后的库名>`(不再接受客户端指定路径,创建时即校验)。 - API:`POST /api/libraries` 只需 `{name}`;`root_path` 由服务端生成为 `BOOKS_DIR/<清洗后的库名>`(不再接受客户端指定路径,创建时即校验)。
- Repo structure conformed to `AGENTS.md`: `web/` renamed to `frontend/`; backend HTTP layer moved from `internal/api` to `cmd/webui/{api,handlers}` (`cmd/server` → `cmd/webui`); README/CHANGELOGs relocated under `docs/` (`README_zh.md` added as Chinese mirror); module-level `.gitignore`s added (`backend/`, `deploy/`); stray root `library/` removed (book files live in `deploy/api/storage/`); debug binaries untracked. - Repo structure conformed to `AGENTS.md`: `web/` renamed to `frontend/`; backend HTTP layer moved from `internal/api` to `cmd/webui/{api,handlers}` (`cmd/server` → `cmd/webui`); README/CHANGELOGs relocated under `docs/` (`README_zh.md` added as Chinese mirror); module-level `.gitignore`s added (`backend/`, `deploy/`); stray root `library/` removed (book files live in `deploy/api/storage/`); debug binaries untracked.
+6
View File
@@ -10,6 +10,12 @@ The format loosely follows Keep a Changelog and can be adapted to the team's hab
### Added / 新增 ### Added / 新增
- Folder-structured comics gain a 目录 sidebar in the CBZ reader: each archive folder (第N話…) becomes a chapter that jumps to its first page, with 上一章/下一章 buttons, a current-chapter counter, and the active row highlighted and scrolled into view — flat archives show no extra UI.
- 按目录组织的漫画在 CBZ 阅读器新增「目录」侧栏:压缩包内每个文件夹(第N話…)即一章,点击直达该话首页;工具条配上一章/下一章与当前章计数,目录内高亮当前章并自动居中;平铺无目录的包不显示多余控件。
- Library uploads over 16MB now auto-switch to the resumable chunked protocol (8MB parts): failed parts retry in place up to 3 times and already-sent parts are skipped, so a flaky upload continues instead of starting over; small files keep the original single request.
- 书库上传超过 16MB 自动切换为可续传分片协议(每片 8MB):失败的分片原地重试至多 3 次,已传片自动跳过,中断后继续而不是从头再来;小文件仍走原单次上传。
- Immersive reading for text & comics (微信读书/Mihon-style): tap the middle of the page to show/hide the reader header and control bar; controls now live in a slide-up bottom sheet — a progress slider jumps to any position (text) or page (CBZ), theme swatches (纸/米/夜) are WYSIWYG color dots, plus A−/A+ font size; body text is serif with a readable justified measure; all prefs persist (localStorage). - Immersive reading for text & comics (微信读书/Mihon-style): tap the middle of the page to show/hide the reader header and control bar; controls now live in a slide-up bottom sheet — a progress slider jumps to any position (text) or page (CBZ), theme swatches (纸/米/夜) are WYSIWYG color dots, plus A−/A+ font size; body text is serif with a readable justified measure; all prefs persist (localStorage).
- 文本与漫画进入沉浸阅读:点正文中央唤出/隐藏顶栏与工具条;控件收进底部滑出抽屉——进度/页码滑条直接跳到全书任意位置或任意页,主题改为所见即所得的色卡(纸/米/夜),保留 A−/A+ 字号;txt/md 正文为衬线、限宽两端对齐;偏好本地持久化。 - 文本与漫画进入沉浸阅读:点正文中央唤出/隐藏顶栏与工具条;控件收进底部滑出抽屉——进度/页码滑条直接跳到全书任意位置或任意页,主题改为所见即所得的色卡(纸/米/夜),保留 A−/A+ 字号;txt/md 正文为衬线、限宽两端对齐;偏好本地持久化。
- CBZ adds reading modes — 连读 (continuous scroll) / 整页 (snap to page top) / 适高 (one page per screen, full width) — cycle button in the bar, choice remembered; tapping the left/right edges turns a page/screen and ←/→/PageUp/PageDown work too. - CBZ adds reading modes — 连读 (continuous scroll) / 整页 (snap to page top) / 适高 (one page per screen, full width) — cycle button in the bar, choice remembered; tapping the left/right edges turns a page/screen and ←/→/PageUp/PageDown work too.
+38 -9
View File
@@ -113,6 +113,33 @@ export function formatPageUrl(fmt: string, n: number): string {
return fmt.replace("%d", String(n)); return fmt.replace("%d", String(n));
} }
const SINGLE_MAX = 16 << 20;
const CHUNK_SIZE = 8 << 20;
const PART_RETRY = 3;
async function uploadChunked(id: number, file: File): Promise<{ accepted: boolean; path: string }> {
const { uploadId } = await apiFetch<{ uploadId: string }>(`/libraries/${id}/upload/init`, {
method: "POST",
body: { name: file.name, size: file.size, chunkSize: CHUNK_SIZE },
});
const { received } = await apiFetch<{ received: number[] }>(`/uploads/${uploadId}`);
const have = new Set(received);
const n = Math.ceil(file.size / CHUNK_SIZE);
for (let i = 0; i < n; i++) {
if (have.has(i)) continue;
for (let tries = 0; ; tries++) {
const res = await fetch(full(`/uploads/${uploadId}/parts/${i}`), {
method: "PUT",
headers: { ...authHeaders(), "Content-Type": "application/octet-stream" },
body: file.slice(i * CHUNK_SIZE, Math.min((i + 1) * CHUNK_SIZE, file.size)),
});
if (res.ok) break;
if (tries >= PART_RETRY) throw await toHttpError(res);
}
}
return apiFetch<{ accepted: boolean; path: string }>(`/uploads/${uploadId}/complete`, { method: "POST" });
}
export const api = { export const api = {
login: (username: string, password: string) => login: (username: string, password: string) =>
apiFetch<{ token: string }>("/auth/login", { method: "POST", body: { username, password } }), apiFetch<{ token: string }>("/auth/login", { method: "POST", body: { username, password } }),
@@ -126,14 +153,15 @@ export const api = {
listLibraries: () => apiFetch<Library[]>("/libraries"), listLibraries: () => apiFetch<Library[]>("/libraries"),
createLibrary: (name: string) => apiFetch<Library>("/libraries", { method: "POST", body: { name } }), createLibrary: (name: string) => apiFetch<Library>("/libraries", { method: "POST", body: { name } }),
scanLibrary: (id: number) => apiFetch<{ accepted: boolean }>(`/libraries/${id}/scan`, { method: "POST" }), scanLibrary: (id: number) => apiFetch<{ accepted: boolean }>(`/libraries/${id}/scan`, { method: "POST" }),
uploadBook: (id: number, file: File) => { // ≤16MB 单 POST;更大走分片协议(init→parts→complete),失败分片原地重试,已传分片跳过
const fd = new FormData(); uploadBook: (id: number, file: File): Promise<{ accepted: boolean; path: string }> =>
fd.append("file", file); file.size <= SINGLE_MAX
return apiFetch<{ accepted: boolean; path: string }>(`/libraries/${id}/upload`, { ? (() => {
method: "POST", const fd = new FormData();
form: fd, fd.append("file", file);
}); return apiFetch<{ accepted: boolean; path: string }>(`/libraries/${id}/upload`, { method: "POST", form: fd });
}, })()
: uploadChunked(id, file),
listBooks: (params: { library?: number; q?: string } = {}) => { listBooks: (params: { library?: number; q?: string } = {}) => {
const sp = new URLSearchParams(); const sp = new URLSearchParams();
@@ -144,7 +172,8 @@ export const api = {
}, },
getBook: (id: number) => apiFetch<Book>(`/books/${id}`), getBook: (id: number) => apiFetch<Book>(`/books/${id}`),
deleteBook: (id: number) => apiFetch<void>(`/books/${id}`, { method: "DELETE" }), deleteBook: (id: number) => apiFetch<void>(`/books/${id}`, { method: "DELETE" }),
pageCount: (pagesUrl: string) => apiFetch<{ count: number }>(pagesUrl), pageCount: (pagesUrl: string) =>
apiFetch<{ count: number; chapters?: { title: string; start: number }[] | null }>(pagesUrl),
putProgress: (id: number, locator: Record<string, unknown>, percent: number, keepalive = false) => putProgress: (id: number, locator: Record<string, unknown>, percent: number, keepalive = false) =>
apiFetch<void>(`/books/${id}/progress`, { method: "PUT", body: { locator, percent }, keepalive }), apiFetch<void>(`/books/${id}/progress`, { method: "PUT", body: { locator, percent }, keepalive }),
+54
View File
@@ -84,6 +84,9 @@ export default function CbzReader({ book, initialLocator, chrome }: ReaderProps)
retry: 0, retry: 0,
}); });
const count = countQ.data?.count ?? 0; const count = countQ.data?.count ?? 0;
const chapters = countQ.data?.chapters ?? null; // 包内目录结构(第N話/上中下卷)
const [toc, setToc] = useState(false);
const activeRow = useRef<HTMLButtonElement>(null);
const saver = useProgressSaver(book.id); const saver = useProgressSaver(book.id);
const boxRef = useRef<HTMLDivElement>(null); const boxRef = useRef<HTMLDivElement>(null);
const [width, setWidth] = useState(480); const [width, setWidth] = useState(480);
@@ -168,6 +171,10 @@ export default function CbzReader({ book, initialLocator, chrome }: ReaderProps)
} }
const cur = ph.pageAt(top + vh / 2); const cur = ph.pageAt(top + vh / 2);
const ci = chapters ? chapters.reduce((acc, c, i) => (c.start <= cur ? i : acc), 0) : 0;
useEffect(() => {
if (toc) activeRow.current?.scrollIntoView({ block: "center" });
}, [toc]);
function jump(i: number, smooth = true) { function jump(i: number, smooth = true) {
const el = boxRef.current; const el = boxRef.current;
@@ -284,6 +291,26 @@ export default function CbzReader({ book, initialLocator, chrome }: ReaderProps)
> >
{MODE_LABEL[mode]} {MODE_LABEL[mode]}
</button> </button>
{chapters && (
<>
<button className="rd-btn" aria-expanded={toc} aria-controls="cbz-toc" onClick={() => setToc((v) => !v)}>
目录
</button>
<span className="text-xs tabular-nums opacity-60">
{ci + 1}/{chapters.length}
</span>
<button className="rd-btn" disabled={ci === 0} onClick={() => jump(chapters[ci - 1].start)}>
← 上一章
</button>
<button
className="rd-btn"
disabled={ci === chapters.length - 1}
onClick={() => jump(chapters[ci + 1].start)}
>
下一章 →
</button>
</>
)}
</div> </div>
</div> </div>
) : ( ) : (
@@ -294,6 +321,33 @@ export default function CbzReader({ book, initialLocator, chrome }: ReaderProps)
<span className="opacity-60">{Math.round(((cur + 1) / Math.max(1, count)) * 100)}%</span> <span className="opacity-60">{Math.round(((cur + 1) / Math.max(1, count)) * 100)}%</span>
</div> </div>
)} )}
{toc && chapters && (
<>
<div className="fixed inset-0 z-10" aria-hidden="true" onClick={() => setToc(false)} />
<nav
id="cbz-toc"
aria-label="章节目录"
className="rd-divider absolute inset-y-0 left-0 z-20 w-64 overflow-y-auto border-r bg-[var(--rd-bg)] p-2 shadow-2xl"
>
<ul className="space-y-0.5">
{chapters.map((c, i) => (
<li key={i}>
<button
ref={i === ci ? activeRow : undefined}
className={"rd-row" + (i === ci ? " rd-btn-on" : "")}
onClick={() => {
jump(c.start);
setToc(false);
}}
>
{i + 1}. {c.title}
</button>
</li>
))}
</ul>
</nav>
</>
)}
</div> </div>
); );
} }