diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..91a646f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: CI +on: + push: + branches: [master, 'fix/**', 'feat/**'] + pull_request: + branches: [master] + +jobs: + backend: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: booklib + POSTGRES_PASSWORD: booklib + POSTGRES_DB: booklib + ports: ['5432:5432'] + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 3s + --health-retries 10 + redis: + image: redis:7 + ports: ['6379:6379'] + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: backend/go.mod + - name: Format check + run: | + cd backend + OUT=$(gofmt -l .) + if [ -n "$OUT" ]; then echo "gofmt violations:"; echo "$OUT"; exit 1; fi + - name: Vet + run: cd backend && go vet ./... + - name: Test + run: cd backend && go test -p 1 -count=1 ./... + env: + DATABASE_URL: postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable + REDIS_URL: redis://localhost:6379/0 + + frontend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: cd frontend && npm ci + - run: cd frontend && npm run check diff --git a/backend/cmd/webui/api/router.go b/backend/cmd/webui/api/router.go index 98fe02a..52e1e5b 100644 --- a/backend/cmd/webui/api/router.go +++ b/backend/cmd/webui/api/router.go @@ -7,14 +7,16 @@ import ( "booklib/cmd/webui/handlers" "booklib/internal/config" - "booklib/internal/redispkg" - "booklib/internal/scanner" - "booklib/internal/store" + "booklib/internal/ports" ) -func NewRouter(cfg *config.Config, st *store.Store, rdb *redispkg.R, sc *scanner.Scanner) *gin.Engine { +// NewRouter 只依赖 port 接口:main.go 传具体实现(*store.Store 满足 5 个 store +// 接口,*redispkg.R 满足 RateLimiter),测试传 portsfake。 +func NewRouter(cfg *config.Config, users ports.UserStore, libs ports.LibraryStore, books ports.BookStore, + progress ports.ProgressStore, bookmarks ports.BookmarkStore, rl ports.RateLimiter, + sc ports.Scanner, med ports.Media, up ports.UploadSessions) *gin.Engine { gin.SetMode(gin.ReleaseMode) - h := handlers.New(cfg, st, rdb, sc) + h := handlers.New(cfg, users, libs, books, progress, bookmarks, rl, sc, med, up) r := gin.New() if e := r.SetTrustedProxies(cfg.TrustedProxies); e != nil { panic(e) @@ -27,17 +29,17 @@ func NewRouter(cfg *config.Config, st *store.Store, rdb *redispkg.R, sc *scanner p := g.Group("", h.AuthMw()) p.GET("/auth/me", h.Me) - users := p.Group("/users", h.AdminOnly()) - users.GET("", h.ListUsers) - users.POST("", h.CreateUser) - users.DELETE("/:id", h.DeleteUser) + usersGrp := p.Group("/users", h.AdminOnly()) + usersGrp.GET("", h.ListUsers) + usersGrp.POST("", h.CreateUser) + usersGrp.DELETE("/:id", h.DeleteUser) - libs := p.Group("/libraries") - libs.GET("", h.ListLibraries) - libs.POST("", h.AdminOnly(), h.CreateLibrary) - libs.POST("/:id/scan", h.AdminOnly(), h.ScanLibrary) - libs.POST("/:id/upload", h.AdminOnly(), h.Upload) - libs.POST("/:id/upload/init", h.AdminOnly(), h.UploadInit) + libsGrp := p.Group("/libraries") + libsGrp.GET("", h.ListLibraries) + libsGrp.POST("", h.AdminOnly(), h.CreateLibrary) + libsGrp.POST("/:id/scan", h.AdminOnly(), h.ScanLibrary) + libsGrp.POST("/:id/upload", h.AdminOnly(), h.Upload) + libsGrp.POST("/:id/upload/init", h.AdminOnly(), h.UploadInit) uploads := p.Group("/uploads", h.AdminOnly()) uploads.GET("/:uid", h.UploadStatus) diff --git a/backend/cmd/webui/api/router_test.go b/backend/cmd/webui/api/router_test.go index 26557cb..2519cfc 100644 --- a/backend/cmd/webui/api/router_test.go +++ b/backend/cmd/webui/api/router_test.go @@ -7,7 +7,6 @@ import ( "time" "booklib/internal/config" - "booklib/internal/redispkg" ) func testCfg() *config.Config { @@ -16,7 +15,7 @@ func testCfg() *config.Config { } func TestHealthz(t *testing.T) { - r := NewRouter(testCfg(), nil, redispkg.New(""), nil) + r := NewRouter(testCfg(), nil, nil, nil, nil, nil, nil, nil, nil, nil) req := httptest.NewRequest(http.MethodGet, "/api/healthz", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) @@ -24,3 +23,56 @@ func TestHealthz(t *testing.T) { t.Fatalf("healthz = %d, want 200", w.Code) } } + +// TestRouterContract 钉死全量路由表(Task 27):增删改任何路由都会使此测试失败, +// 必须显式更新——路由即对外契约。 +func TestRouterContract(t *testing.T) { + r := NewRouter(testCfg(), nil, nil, nil, nil, nil, nil, nil, nil, nil) + want := map[string]bool{ + "GET /api/healthz": true, + "POST /api/auth/login": true, + "GET /api/auth/me": true, + "GET /api/users": true, + "POST /api/users": true, + "DELETE /api/users/:id": true, + "GET /api/libraries": true, + "POST /api/libraries": true, + "POST /api/libraries/:id/scan": true, + "POST /api/libraries/:id/upload": true, + "POST /api/libraries/:id/upload/init": true, + "GET /api/uploads/:uid": true, + "POST /api/uploads/:uid/complete": true, + "PUT /api/uploads/:uid/parts/:index": true, + "GET /api/books": true, + "GET /api/books/:id": true, + "DELETE /api/books/:id": true, + "GET /api/books/:id/cover": true, + "GET /api/books/:id/file": true, + "GET /api/books/:id/pages": true, + "GET /api/books/:id/pages/:n": true, + "PUT /api/books/:id/progress": true, + "GET /api/progress": true, + "GET /api/books/:id/bookmarks": true, + "POST /api/books/:id/bookmarks": true, + "PATCH /api/bookmarks/:id": true, + "DELETE /api/bookmarks/:id": true, + } + got := map[string]bool{} + for _, rt := range r.Routes() { + key := rt.Method + " " + rt.Path + if got[key] { + t.Errorf("duplicate route %s", key) + } + got[key] = true + } + for k := range want { + if !got[k] { + t.Errorf("missing route %s", k) + } + } + for k := range got { + if !want[k] { + t.Errorf("unexpected route %s", k) + } + } +} diff --git a/backend/cmd/webui/handlers/auth.go b/backend/cmd/webui/handlers/auth.go index 641c26c..2ba9511 100644 --- a/backend/cmd/webui/handlers/auth.go +++ b/backend/cmd/webui/handlers/auth.go @@ -20,11 +20,11 @@ func (h *H) Login(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "username and password required") return } - if n := h.rdb.IncrWindow(c, "loginrl:"+c.ClientIP(), loginWindow); n > loginMax { + if n := h.rl.IncrWindow(c, "loginrl:"+c.ClientIP(), loginWindow); n > loginMax { err(c, http.StatusTooManyRequests, "rate_limited", "too many login attempts") return } - u, qerr := h.st.GetUserByName(c, req.Username) + u, qerr := h.users.GetUserByName(c, req.Username) if qerr != nil { if !errors.Is(qerr, pgx.ErrNoRows) { dbErr(c, qerr) @@ -48,9 +48,14 @@ func (h *H) Login(c *gin.Context) { } func (h *H) Me(c *gin.Context) { - u, qerr := h.st.GetUserByID(c, uid(c)) + // B7: only no-rows → 401; other errors (PG down) go through dbErr → 503. + u, qerr := h.users.GetUserByID(c, uid(c)) if qerr != nil { - err(c, http.StatusUnauthorized, "unauthorized", "no such user") + if errors.Is(qerr, pgx.ErrNoRows) { + err(c, http.StatusUnauthorized, "unauthorized", "no such user") + return + } + dbErr(c, qerr) return } c.JSON(http.StatusOK, gin.H{"id": u.ID, "username": u.Username, "role": u.Role}) diff --git a/backend/cmd/webui/handlers/auth_test.go b/backend/cmd/webui/handlers/auth_test.go index 829df80..94222b5 100644 --- a/backend/cmd/webui/handlers/auth_test.go +++ b/backend/cmd/webui/handlers/auth_test.go @@ -18,9 +18,11 @@ import ( "booklib/internal/auth" "booklib/internal/config" "booklib/internal/db" + "booklib/internal/media" "booklib/internal/redispkg" "booklib/internal/scanner" "booklib/internal/store" + "booklib/internal/upload" ) func testCfg() *config.Config { @@ -57,8 +59,10 @@ func setupAPI(t *testing.T) (*store.Store, *scanner.Scanner, http.Handler, strin cfg.BooksDir = booksDir cfg.CacheDir = t.TempDir() rdb := redispkg.New(os.Getenv("REDIS_URL")) - sc := scanner.New(st, cfg, rdb) - r := api.NewRouter(cfg, st, rdb, sc) + med := media.New(cfg, rdb) + up := upload.New(cfg.BooksDir, cfg.UploadMaxMB) + sc := scanner.New(st, cfg, rdb, up) + r := api.NewRouter(cfg, st, st, st, st, st, rdb, sc, med, up) if u := os.Getenv("REDIS_URL"); u != "" { // 测试卫生: 共享 redis 上重置登录限流桶, 防跨测试累计 429 if opt, e := redis.ParseURL(u); e == nil { rc := redis.NewClient(opt) diff --git a/backend/cmd/webui/handlers/bookmarks.go b/backend/cmd/webui/handlers/bookmarks.go index 507f74d..454f391 100644 --- a/backend/cmd/webui/handlers/bookmarks.go +++ b/backend/cmd/webui/handlers/bookmarks.go @@ -27,7 +27,7 @@ func (h *H) ListBookmarks(c *gin.Context) { if !ok { return } - rows, e := h.st.ListBookmarks(c, uid(c), b.LibraryID, b.Path) + rows, e := h.bookmarks.ListBookmarks(c, uid(c), b.LibraryID, b.Path) if e != nil { dbErr(c, e) return @@ -65,7 +65,7 @@ func (h *H) CreateBookmark(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "note too long (max 500 characters)") return } - id, e := h.st.InsertBookmark(c, uid(c), b.LibraryID, b.Path, req.Locator, req.Percent, req.Note) + id, e := h.bookmarks.InsertBookmark(c, uid(c), b.LibraryID, b.Path, req.Locator, req.Percent, req.Note) if e != nil { dbErr(c, e) return @@ -101,7 +101,7 @@ func (h *H) PatchBookmark(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "note too long (max 500 characters)") return } - updated, e := h.st.UpdateBookmarkNote(c, uid(c), id, req.Note) + updated, e := h.bookmarks.UpdateBookmarkNote(c, uid(c), id, req.Note) if e != nil { dbErr(c, e) return @@ -118,7 +118,7 @@ func (h *H) DeleteBookmark(c *gin.Context) { if !ok { return } - deleted, e := h.st.DeleteBookmark(c, uid(c), id) + deleted, e := h.bookmarks.DeleteBookmark(c, uid(c), id) if e != nil { dbErr(c, e) return diff --git a/backend/cmd/webui/handlers/books.go b/backend/cmd/webui/handlers/books.go index 1455bb1..b23c237 100644 --- a/backend/cmd/webui/handlers/books.go +++ b/backend/cmd/webui/handlers/books.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "strconv" - "strings" "time" "github.com/gin-gonic/gin" @@ -18,7 +17,7 @@ import ( ) func (h *H) getBookRow(c *gin.Context, id int64) (store.Book, bool) { - b, e := h.st.GetBook(c, id) + b, e := h.books.GetBook(c, id) if e != nil { if errors.Is(e, pgx.ErrNoRows) { err(c, http.StatusNotFound, "not_found", "no such book") @@ -31,41 +30,23 @@ func (h *H) getBookRow(c *gin.Context, id int64) (store.Book, bool) { } func (h *H) bookFromParam(c *gin.Context) (store.Book, bool) { - id, e := strconv.ParseInt(c.Param("id"), 10, 64) - if e != nil { - err(c, http.StatusBadRequest, "bad_request", "bad id") + id, ok := idParam(c) + if !ok { return store.Book{}, false } return h.getBookRow(c, id) } -func (h *H) getLibRow(c *gin.Context, id int64) (store.Library, bool) { - l, e := h.st.GetLibrary(c, id) - if e != nil { - if errors.Is(e, pgx.ErrNoRows) { - err(c, http.StatusNotFound, "not_found", "no such library") - return store.Library{}, false - } - dbErr(c, e) - return store.Library{}, false - } - return l, true -} - -// absBookPath: books.path 永远相对且不含 ..;拼接后二次前缀校验(纵深防御) +// absBookPath: books.path 永远相对且不含 ..;拼接后二次前缀校验(纵深防御)。 +// bookfile.Contains 带 EvalSymlinks,比裸 filepath.Rel 更能拦住软链逃逸。 func absBookPath(root string, b store.Book) (string, error) { abs := filepath.Join(root, filepath.FromSlash(b.Path)) - if filepath.Clean(abs) != abs || !hasPrefixDir(abs, root) { + if filepath.Clean(abs) != abs || !bookfile.Contains(root, abs) { return "", os.ErrPermission } return abs, nil } -func hasPrefixDir(p, dir string) bool { - rel, err := filepath.Rel(filepath.Clean(dir), filepath.Clean(p)) - return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) -} - func bookJSON(b store.Book, percent float64, libraryName string) gin.H { h := bookfile.Hash(b.FileSize, b.ModTS) j := gin.H{ @@ -89,7 +70,7 @@ func bookJSON(b store.Book, percent float64, libraryName string) gin.H { func (h *H) ListBooks(c *gin.Context) { libID, _ := strconv.ParseInt(c.Query("library"), 10, 64) - views, e := h.st.ListBooks(c, libID, c.Query("q"), c.Query("prefix"), uid(c)) + views, e := h.books.ListBooks(c, libID, c.Query("q"), c.Query("prefix"), uid(c)) if e != nil { dbErr(c, e) return @@ -106,12 +87,12 @@ func (h *H) GetBook(c *gin.Context) { if !ok { return } - p, e := h.st.GetProgress(c, uid(c), b.LibraryID, b.Path) // ErrNoRows → 零值 percent + p, e := h.progress.GetProgress(c, uid(c), b.LibraryID, b.Path) // ErrNoRows → 零值 percent if e != nil && !errors.Is(e, pgx.ErrNoRows) { dbErr(c, e) return } - lib, e := h.st.GetLibrary(c, b.LibraryID) + lib, e := h.libs.GetLibrary(c, b.LibraryID) if e != nil && !errors.Is(e, pgx.ErrNoRows) { // 库被并发删则留空 library 名,书仍可见 dbErr(c, e) return @@ -124,7 +105,7 @@ func (h *H) DeleteBook(c *gin.Context) { if !ok { return } - lib, ok := h.getLibRow(c, b.LibraryID) + lib, ok := h.getLib(c, b.LibraryID) if !ok { return } @@ -144,7 +125,7 @@ func (h *H) DeleteBook(c *gin.Context) { key := bookfile.DirKey(b.ID, bookfile.Hash(b.FileSize, b.ModTS)) os.RemoveAll(bookfile.CoverDir(h.cfg.CacheDir, key)) os.RemoveAll(bookfile.PagesDir(h.cfg.CacheDir, key)) - if e := h.st.DeleteBook(c, b.ID); e != nil { + if e := h.books.DeleteBook(c, b.ID); e != nil { dbErr(c, e) return } diff --git a/backend/cmd/webui/handlers/content.go b/backend/cmd/webui/handlers/content.go index f8e03c5..34fe433 100644 --- a/backend/cmd/webui/handlers/content.go +++ b/backend/cmd/webui/handlers/content.go @@ -1,14 +1,13 @@ package handlers import ( - "fmt" + "errors" + "io/fs" "net/http" "os" - "path" "path/filepath" "strconv" "strings" - "time" "github.com/gin-gonic/gin" @@ -19,7 +18,7 @@ import ( const defaultCover = `` func (h *H) bookRoot(c *gin.Context, b store.Book) (string, bool) { - lib, ok := h.getLibRow(c, b.LibraryID) + lib, ok := h.getLib(c, b.LibraryID) if !ok { return "", false } @@ -30,6 +29,26 @@ func (h *H) immutable(c *gin.Context) { c.Header("Cache-Control", "public, max-age=31536000, immutable") } +// checkPath 纵深防御:path 越界 → 403,与原契约一致。 +func checkPath(c *gin.Context, root string, b store.Book) (string, bool) { + abs, perr := absBookPath(root, b) + if perr != nil { + err(c, http.StatusForbidden, "forbidden", "unsafe path") + return "", false + } + return abs, true +} + +// mapContentErr 把 media/bookfile 的文件级失败映射回原契约状态码: +// 文件不在盘上 → 404,其余(坏包等)由调用方决定 422/500。 +func mapContentErr(c *gin.Context, e error) bool { + if errors.Is(e, fs.ErrNotExist) { + err(c, http.StatusNotFound, "not_found", "file missing on disk") + return true + } + return false +} + func (h *H) ServeCover(c *gin.Context) { b, ok := h.bookFromParam(c) if !ok { @@ -47,32 +66,30 @@ func (h *H) ServeCover(c *gin.Context) { } if b.Format == "cbz" || b.Format == "epub" { // 自愈:缓存丢了就地抽封面(重启/卷漂移/扫描器还没跑到) if root, ok := h.bookRoot(c, b); ok { - if f, size, ok := h.openBook(c, b, root); ok { - defer f.Close() - var img []byte - var ext string - var e error - if b.Format == "cbz" { - img, ext, e = bookfile.CBZCover(f, size) - } else { - img, ext, e = bookfile.EPUBCover(f, size) - } - if e == nil { - dst := filepath.Join(dir, "cover"+ext) - if e := os.MkdirAll(dir, 0o755); e == nil { - tmp := fmt.Sprintf("%s.tmp-%d", dst, time.Now().UnixNano()) // 并发幂等:唯一 tmp + rename 原子 - if e := os.WriteFile(tmp, img, 0o644); e == nil { - if e := os.Rename(tmp, dst); e == nil { - http.ServeFile(c.Writer, c.Request, dst) + if _, ok := checkPath(c, root, b); ok { + e := h.med.EnsureCover(c, b.ID, b.Format, b.FileSize, b.ModTS, root, b.Path) + switch { + case e == nil: + if entries, re := os.ReadDir(dir); re == nil { + for _, en := range entries { + if !strings.Contains(en.Name(), ".tmp") { + http.ServeFile(c.Writer, c.Request, filepath.Join(dir, en.Name())) + return } } - os.Remove(tmp) } + case mapContentErr(c, e): + return // 404 已回复 } + // 抽取失败(坏包)→ 落到占位 SVG,与原契约一致 + } else { + return // 403 已回复 } + } else { + return // 404/503 已回复 } } - if c.Writer.Written() { // openBook/bookRoot 已写 403/404/500,不再叠加占位图 + if c.Writer.Written() { return } c.Data(http.StatusOK, "image/svg+xml", []byte(defaultCover)) @@ -87,55 +104,32 @@ func (h *H) ServeFile(c *gin.Context) { if !ok { return } - abs, perr := absBookPath(root, b) - if perr != nil { - err(c, http.StatusForbidden, "forbidden", "unsafe path") + abs, ok := checkPath(c, root, b) + if !ok { return } - c.Header("ETag", `"`+bookfile.Hash(b.FileSize, b.ModTS)+`"`) + if _, e := os.Stat(abs); e != nil { + err(c, http.StatusNotFound, "not_found", "file missing on disk") + return + } + c.Header("ETag", `"`+h.med.CacheBuster(b.FileSize, b.ModTS)+`"`) c.Header("Cache-Control", "private, must-revalidate") http.ServeFile(c.Writer, c.Request, abs) } -func (h *H) openBook(c *gin.Context, b store.Book, root string) (*os.File, int64, bool) { - abs, perr := absBookPath(root, b) - if perr != nil { - err(c, http.StatusForbidden, "forbidden", "unsafe path") - return nil, 0, false +// pageIndex 走 media(redis 缓存 + 索引提取);路径校验仍在 handler,保住 403 契约。 +func (h *H) pageIndex(c *gin.Context, b store.Book, root string) ([]string, bool) { + if _, ok := checkPath(c, root, b); !ok { + return nil, false } - f, perr := os.Open(abs) - if perr != nil { - err(c, http.StatusNotFound, "not_found", "file missing on disk") - return nil, 0, false - } - st, perr := f.Stat() - if perr != nil { - f.Close() - err(c, http.StatusInternalServerError, "internal", "stat") - return nil, 0, false - } - return f, st.Size(), true -} - -func (h *H) pageIndex(c *gin.Context, b store.Book, root string) ([]string, error) { - hash := bookfile.Hash(b.FileSize, b.ModTS) - key := fmt.Sprintf("pagesidx2:%d:%s", b.ID, hash) // 前缀换代 = 索引逻辑变更时一次性作废旧缓存 - if v, ok := h.rdb.Get(c, key); ok && v != "" { - return strings.Split(v, "\n"), nil - } - f, size, ok := h.openBook(c, b, root) - if !ok { - return nil, os.ErrNotExist - } - defer f.Close() - idx, e := bookfile.PageIndex(f, size) + idx, e := h.med.PageIndex(c, b.ID, b.FileSize, b.ModTS, root, b.Path) if e != nil { - return nil, e + if !mapContentErr(c, e) { + err(c, http.StatusUnprocessableEntity, "broken", e.Error()) + } + return nil, false } - if len(idx) > 0 { // 空索引不缓存,否则 warm 命中 "" 会 Split 出幽灵页 - h.rdb.Set(c, key, strings.Join(idx, "\n"), 7*24*time.Hour) - } - return idx, nil + return idx, true } func (h *H) PagesCount(c *gin.Context) { @@ -151,56 +145,11 @@ func (h *H) PagesCount(c *gin.Context) { if !ok { return } - idx, e := h.pageIndex(c, b, root) - if e != nil { - if c.Writer.Written() { - return // openBook 已写 403/404,不再叠加 422 - } - err(c, http.StatusUnprocessableEntity, "broken", e.Error()) + idx, ok := h.pageIndex(c, b, root) + if !ok { return } - c.JSON(http.StatusOK, gin.H{"count": len(idx), "chapters": chaptersOf(idx)}) -} - -type cbzChapter struct { - Title string `json:"title"` - Start int `json:"start"` -} - -// chaptersOf 页索引已自然序排列,按父目录分组:每个新目录开一章,标题取目录名;扁平包或只有一组返回 nil -func chaptersOf(idx []string) []cbzChapter { - type grp struct { - dir string - start int - } - var grps []grp - last := "\x00" - for i, n := range idx { - d := path.Dir(n) - if d == last { - continue - } - last = d - if d == "." { // 根目录散页不开章 - continue - } - grps = append(grps, grp{d, i}) - } - if len(grps) < 2 { - return nil - } - titles := make(map[string]int) - out := make([]cbzChapter, len(grps)) - for i, g := range grps { - out[i] = cbzChapter{Title: path.Base(g.dir), Start: g.start} - titles[out[i].Title]++ - } - for i, g := range grps { // 同名目录(不同父级)撞车 → 用全路径消歧 - if titles[out[i].Title] > 1 { - out[i].Title = g.dir - } - } - return out + c.JSON(http.StatusOK, gin.H{"count": len(idx), "chapters": h.med.ChaptersOf(idx)}) } func (h *H) Page(c *gin.Context) { @@ -221,47 +170,22 @@ func (h *H) Page(c *gin.Context) { if !ok { return } - idx, e := h.pageIndex(c, b, root) - if e != nil { - if c.Writer.Written() { - return // openBook 已写 403/404,不再叠加 422 - } - err(c, http.StatusUnprocessableEntity, "broken", e.Error()) + idx, ok := h.pageIndex(c, b, root) + if !ok { return } if n >= len(idx) { err(c, http.StatusNotFound, "not_found", "no such page") return } - ext := strings.ToLower(filepath.Ext(idx[n])) - dir := bookfile.PagesDir(h.cfg.CacheDir, bookfile.DirKey(b.ID, bookfile.Hash(b.FileSize, b.ModTS))) - dst := filepath.Join(dir, strconv.Itoa(n)+ext) - if _, e := os.Stat(dst); e != nil { // miss → 解压落盘(并发重做同页幂等,唯一 tmp 名 + rename 原子) - f, size, ok := h.openBook(c, b, root) - if !ok { - return - } - defer f.Close() - data, e := bookfile.ReadEntry(f, size, idx[n]) - if e != nil { - err(c, http.StatusInternalServerError, "internal", "extract page") - return - } - if e := os.MkdirAll(dir, 0o755); e != nil { - err(c, http.StatusInternalServerError, "internal", "cache dir") - return - } - tmp := fmt.Sprintf("%s.tmp-%d", dst, time.Now().UnixNano()) - if e := os.WriteFile(tmp, data, 0o644); e != nil { - os.Remove(tmp) - err(c, http.StatusInternalServerError, "internal", "write cache") - return - } - if e := os.Rename(tmp, dst); e != nil { - os.Remove(tmp) - err(c, http.StatusInternalServerError, "internal", "rename cache") + // miss → 解压落盘(media 内部唯一 tmp 名 + rename 原子,并发重做同页幂等) + dst, e := h.med.EnsurePage(c, b.ID, b.FileSize, b.ModTS, root, b.Path, n, idx) + if e != nil { + if mapContentErr(c, e) { return } + err(c, http.StatusInternalServerError, "internal", "extract page") + return } h.immutable(c) http.ServeFile(c.Writer, c.Request, dst) diff --git a/backend/cmd/webui/handlers/content_test.go b/backend/cmd/webui/handlers/content_test.go index 93f5436..6129753 100644 --- a/backend/cmd/webui/handlers/content_test.go +++ b/backend/cmd/webui/handlers/content_test.go @@ -126,9 +126,9 @@ func TestPagesChaptersAndNoImageZip(t *testing.T) { tok := adminToken(t, h) lib, root := newLibrary(t, st, h, tok, booksDir, "ch") zipTo(t, filepath.Join(root, "show.cbz"), map[string]string{ - "第2季/第2話/0001.jpg": "IMG2", - "第2季/第1話/0001.jpg": "IMG1a", - "第2季/第1話/._0001.jpg": "junk", + "第2季/第2話/0001.jpg": "IMG2", + "第2季/第1話/0001.jpg": "IMG1a", + "第2季/第1話/._0001.jpg": "junk", "__MACOSX/第2季/._0001.jpg": "junk", }) zipTo(t, filepath.Join(root, "videos.zip"), map[string]string{"ep/01.mkv": "x"}) diff --git a/backend/cmd/webui/handlers/handlers.go b/backend/cmd/webui/handlers/handlers.go index e5a0633..ee07e8d 100644 --- a/backend/cmd/webui/handlers/handlers.go +++ b/backend/cmd/webui/handlers/handlers.go @@ -6,6 +6,7 @@ import ( "log" "net" "net/http" + "strconv" "strings" "syscall" @@ -15,26 +16,45 @@ import ( "booklib/internal/auth" "booklib/internal/config" - "booklib/internal/redispkg" - "booklib/internal/scanner" - "booklib/internal/store" + "booklib/internal/ports" ) +// H 只依赖 ports 里的小口径接口(Task 25):具体实现由 main.go 装配, +// 测试可注入手写 fake(portsfake),无需 PG/Redis。 type H struct { - cfg *config.Config - st *store.Store - rdb *redispkg.R - sc *scanner.Scanner + cfg *config.Config + users ports.UserStore + libs ports.LibraryStore + books ports.BookStore + progress ports.ProgressStore + bookmarks ports.BookmarkStore + rl ports.RateLimiter + sc ports.Scanner + med ports.Media + up ports.UploadSessions } -func New(cfg *config.Config, st *store.Store, rdb *redispkg.R, sc *scanner.Scanner) *H { - return &H{cfg: cfg, st: st, rdb: rdb, sc: sc} +func New(cfg *config.Config, users ports.UserStore, libs ports.LibraryStore, books ports.BookStore, + progress ports.ProgressStore, bookmarks ports.BookmarkStore, rl ports.RateLimiter, + sc ports.Scanner, med ports.Media, up ports.UploadSessions) *H { + return &H{cfg: cfg, users: users, libs: libs, books: books, progress: progress, + bookmarks: bookmarks, rl: rl, sc: sc, med: med, up: up} } func err(c *gin.Context, status int, code, msg string) { c.AbortWithStatusJSON(status, gin.H{"error": gin.H{"code": code, "message": msg}}) } +// idParam 解析 :id 路径参数,失败已回复 400。 +func idParam(c *gin.Context) (int64, bool) { + id, e := strconv.ParseInt(c.Param("id"), 10, 64) + if e != nil { + err(c, http.StatusBadRequest, "bad_request", "bad id") + return 0, false + } + return id, true +} + // dbErr 统一处理 store 层失败:记日志;连接类错误 503(Service Unavailable),其余 500 // 注:brief 里的 pgxpool.ErrClosedPool 在 pgx v5 不存在,实际由 puddle 原样透出,用它替代; // PG 停机时池内连接先收到 SQLSTATE 57P01(administrator shutdown),故把 08xx/57Pxx 也归为 503 diff --git a/backend/cmd/webui/handlers/libraries.go b/backend/cmd/webui/handlers/libraries.go index 7edb9a1..b576c71 100644 --- a/backend/cmd/webui/handlers/libraries.go +++ b/backend/cmd/webui/handlers/libraries.go @@ -15,6 +15,8 @@ import ( "github.com/jackc/pgx/v5" "booklib/internal/bookfile" + "booklib/internal/media" + "booklib/internal/ports" "booklib/internal/store" ) @@ -22,7 +24,7 @@ import ( func (h *H) libRoot(c *gin.Context, lib store.Library) (string, bool) { root := filepath.Clean(lib.RootPath) books := filepath.Clean(h.cfg.BooksDir) - if !filepath.IsAbs(root) || (root != books && !strings.HasPrefix(root, books+string(os.PathSeparator))) { + if !filepath.IsAbs(root) || !bookfile.Contains(books, root) { err(c, http.StatusForbidden, "forbidden", "library root outside books dir") return "", false } @@ -33,8 +35,22 @@ func (h *H) libRoot(c *gin.Context, lib store.Library) (string, bool) { return root, true } +// getLib 查库行,404/503/500 已回复(原 getLibrary/getLibRow 合一,Task 25) +func (h *H) getLib(c *gin.Context, id int64) (store.Library, bool) { + l, e := h.libs.GetLibrary(c, id) + if e != nil { + if errors.Is(e, pgx.ErrNoRows) { + err(c, http.StatusNotFound, "not_found", "no such library") + return store.Library{}, false + } + dbErr(c, e) + return store.Library{}, false + } + return l, true +} + func (h *H) ListLibraries(c *gin.Context) { - libs, e := h.st.ListLibraries(c) + libs, e := h.libs.ListLibraries(c) if e != nil { dbErr(c, e) return @@ -61,10 +77,15 @@ func (h *H) CreateLibrary(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "bad name") return } + // B8: reject reserved names that conflict with system directories. + if media.IsReservedName(safe) { + err(c, http.StatusBadRequest, "bad_request", "reserved_name") + return + } root := filepath.Join(filepath.Clean(h.cfg.BooksDir), safe) - id, e := h.st.CreateLibrary(c, req.Name, root) + id, e := h.libs.CreateLibrary(c, req.Name, root) if e != nil { - if isUnique(e) { + if ports.IsUniqueViolation(e) { err(c, http.StatusConflict, "exists", "name taken") return } @@ -74,26 +95,12 @@ func (h *H) CreateLibrary(c *gin.Context) { c.JSON(http.StatusCreated, gin.H{"id": id, "name": req.Name, "root_path": root}) } -func (h *H) getLibrary(c *gin.Context) (store.Library, bool) { - id, e := strconv.ParseInt(c.Param("id"), 10, 64) - if e != nil { - err(c, http.StatusBadRequest, "bad_request", "bad id") - return store.Library{}, false - } - lib, e := h.st.GetLibrary(c, id) - if e != nil { - if errors.Is(e, pgx.ErrNoRows) { - err(c, http.StatusNotFound, "not_found", "no such library") - return store.Library{}, false - } - dbErr(c, e) - return store.Library{}, false - } - return lib, true -} - func (h *H) ScanLibrary(c *gin.Context) { - lib, ok := h.getLibrary(c) + id, ok := idParam(c) + if !ok { + return + } + lib, ok := h.getLib(c, id) if !ok { return } @@ -105,7 +112,11 @@ func (h *H) ScanLibrary(c *gin.Context) { } func (h *H) Upload(c *gin.Context) { - lib, ok := h.getLibrary(c) + id, ok := idParam(c) + if !ok { + return + } + lib, ok := h.getLib(c, id) if !ok { return } @@ -129,27 +140,51 @@ func (h *H) Upload(c *gin.Context) { err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md") return } - dst, e := h.uniquePath(root, name) - if e != nil { - err(c, http.StatusForbidden, "forbidden", e.Error()) + // B12: retry on O_EXCL collision — concurrent uploads with the same name + // can both get the same candidate from UniquePath (stat-then-create race). + var dst, tmp string + var out *os.File + for attempt := 0; attempt < 5; attempt++ { + var e error + dst, e = h.up.UniquePath(root, name) + if e != nil { + err(c, http.StatusForbidden, "forbidden", e.Error()) + return + } + tmp = dst + ".upload-" + strconv.FormatInt(time.Now().UnixNano(), 36) + out, e = os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if e == nil { + break + } + if !os.IsExist(e) { + err(c, http.StatusInternalServerError, "internal", "create tmp") + return + } + // O_EXCL collision — retry with fresh UniquePath. + } + if out == nil { + err(c, http.StatusConflict, "conflict", "too many concurrent uploads with same name") return } src, e := fh.Open() if e != nil { + out.Close() + os.Remove(tmp) err(c, http.StatusInternalServerError, "internal", "open upload") return } defer src.Close() - tmp := dst + ".upload-" + strconv.FormatInt(time.Now().UnixNano(), 36) - out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if e != nil { - err(c, http.StatusInternalServerError, "internal", "create tmp") - return - } + // B6: only MaxBytesError returns 413; other io.Copy failures (disk full, + // connection drop) return 500. if _, e := io.Copy(out, src); e != nil { out.Close() os.Remove(tmp) - err(c, http.StatusRequestEntityTooLarge, "too_large", "upload failed") + var mbe *http.MaxBytesError + if errors.As(e, &mbe) { + err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit") + return + } + err(c, http.StatusInternalServerError, "internal", "upload failed") return } out.Close() @@ -160,28 +195,3 @@ func (h *H) Upload(c *gin.Context) { } c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": strings.TrimPrefix(dst, root+string(os.PathSeparator))}) } - -// uniquePath 清洗后的 name 必须仍在 root 内;重名加 " (n)" 后缀 -func (h *H) uniquePath(root, name string) (string, error) { - ext := filepath.Ext(name) - base := strings.TrimSuffix(name, ext) - for i := 0; ; i++ { - cand := base + ext - if i > 0 { - cand = base + " (" + strconv.Itoa(i) + ")" + ext - } - p := filepath.Join(root, cand) - if filepath.Clean(p) != filepath.Join(root, filepath.Clean(cand)) || - !strings.HasPrefix(filepath.Clean(p), root+string(os.PathSeparator)) { - return "", os.ErrInvalid - } - if _, e := os.Stat(p); os.IsNotExist(e) { - return p, nil - } else if e != nil { - return "", e - } - if i > 999 { - return "", os.ErrExist - } - } -} diff --git a/backend/cmd/webui/handlers/libraries_unit_test.go b/backend/cmd/webui/handlers/libraries_unit_test.go new file mode 100644 index 0000000..9a9c029 --- /dev/null +++ b/backend/cmd/webui/handlers/libraries_unit_test.go @@ -0,0 +1,232 @@ +package handlers_test + +import ( + "net/http" + "strconv" + "strings" + "testing" + "time" + + "booklib/internal/ports" + "booklib/internal/upload" +) + +// ---------- libraries (Task 27, portsfake) ---------- + +func TestUnit_ListLibraries(t *testing.T) { + e := newTestEnv(t) + e.seedLib(t, "comics") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodGet, "/api/libraries", atok, nil) + httpOK(t, w, 200, "list libraries") + var libs []map[string]any + if err := jsonUnmarshal(w, &libs); err != nil { + t.Fatal(err) + } + if len(libs) != 1 || libs[0]["name"] != "comics" { + t.Fatalf("unexpected libs %v", libs) + } +} + +func TestUnit_CreateLibrary_ReservedName(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + // 保留名清单以 media/reserved.go 为准(B8);大小写不敏感。 + // 契约(与重构前一致): code="bad_request", message="reserved_name" + for _, name := range []string{"cache", ".uploads", ".trash", "CACHE", "Cache"} { + w := e.do(t, http.MethodPost, "/api/libraries", atok, map[string]string{"Name": name}) + if w.Code != 400 || errCode(t, w) != "bad_request" || !strings.Contains(w.Body.String(), "reserved_name") { + t.Fatalf("name=%q want 400/bad_request/reserved_name got %d %s", name, w.Code, w.Body.String()) + } + } +} + +func TestUnit_CreateLibrary_BadName(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + for _, name := range []string{"", "..", "///"} { + w := e.do(t, http.MethodPost, "/api/libraries", atok, map[string]string{"Name": name}) + httpOK(t, w, 400, "bad name "+name) + } +} + +func TestUnit_CreateLibrary_Duplicate(t *testing.T) { + e := newTestEnv(t) + e.seedLib(t, "dup") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries", atok, map[string]string{"Name": "dup"}) + httpOK(t, w, 409, "duplicate library") + if code := errCode(t, w); code != "exists" { + t.Fatalf("error code want exists got %q", code) + } +} + +func TestUnit_CreateLibrary_OK(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries", atok, map[string]string{"Name": "new lib"}) + httpOK(t, w, 201, "create library") + body := jsonBody(t, w) + if body["name"] != "new lib" { + t.Fatalf("unexpected body %v", body) + } +} + +func TestUnit_ScanLibrary_NotFound(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries/999/scan", atok, nil) + httpOK(t, w, 404, "scan missing library") +} + +func TestUnit_ScanLibrary_Accepted(t *testing.T) { + e := newTestEnv(t) + libID, _ := e.seedLib(t, "scannable") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries/"+strconv.FormatInt(libID, 10)+"/scan", atok, nil) + httpOK(t, w, 202, "scan accepted") + if !e.sc.WaitForScan(1, time.Second) { + t.Fatalf("ScanLibraryByID never called; seen=%v", e.sc.Seen()) + } + if seen := e.sc.Seen(); seen[0] != libID { + t.Fatalf("scanned lib %d want %d", seen[0], libID) + } +} + +func TestUnit_ScanLibrary_MemberForbidden(t *testing.T) { + e := newTestEnv(t) + libID, _ := e.seedLib(t, "nope") + mtok := e.token(t, "member", 2) + w := e.do(t, http.MethodPost, "/api/libraries/"+strconv.FormatInt(libID, 10)+"/scan", mtok, nil) + httpOK(t, w, 403, "member scan") +} + +// ---------- uploads: sentinel → status mapping ---------- + +func TestUnit_UploadInit_MapsSentinels(t *testing.T) { + libIDPath := func(e *testEnv) string { + id, _ := e.seedLib(t, "up") + return "/api/libraries/" + strconv.FormatInt(id, 10) + "/upload/init" + } + cases := []struct { + name string + initErr error + wantCode int + wantErr string + }{ + {"too large", ports.ErrTooLarge, 413, "too_large"}, + {"bad name", upload.ErrBadName, 400, "bad_request"}, + {"bad format", upload.ErrBadFormat, 400, "bad_format"}, + {"bad size", upload.ErrBadSize, 400, "bad_request"}, + {"bad chunk", upload.ErrBadChunk, 400, "bad_request"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := newTestEnv(t) + e.up.InitErr = tc.initErr + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, libIDPath(e), atok, + map[string]any{"name": "b.cbz", "size": 10, "chunkSize": 4}) + httpOK(t, w, tc.wantCode, "init "+tc.name) + if code := errCode(t, w); code != tc.wantErr { + t.Fatalf("error code want %q got %q", tc.wantErr, code) + } + }) + } +} + +func TestUnit_UploadInit_OK(t *testing.T) { + e := newTestEnv(t) + id, _ := e.seedLib(t, "up") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries/"+strconv.FormatInt(id, 10)+"/upload/init", atok, + map[string]any{"name": "b.cbz", "size": 10, "chunkSize": 4}) + httpOK(t, w, 200, "init ok") + body := jsonBody(t, w) + if body["uploadId"] != e.up.UID { + t.Fatalf("uploadId want %q got %v", e.up.UID, body["uploadId"]) + } +} + +func TestUnit_UploadInit_BadBody(t *testing.T) { + e := newTestEnv(t) + id, _ := e.seedLib(t, "up") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/libraries/"+strconv.FormatInt(id, 10)+"/upload/init", atok, nil) + httpOK(t, w, 400, "init without json body") +} + +func TestUnit_UploadStatus_NotFound(t *testing.T) { + e := newTestEnv(t) + e.up.StatusErr = ports.ErrNotFound + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodGet, "/api/uploads/"+e.up.UID, atok, nil) + httpOK(t, w, 404, "status missing upload") + if code := errCode(t, w); code != "not_found" { + t.Fatalf("error code want not_found got %q", code) + } +} + +func TestUnit_UploadStatus_OK(t *testing.T) { + e := newTestEnv(t) + e.up.Received = []int64{0, 2} + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodGet, "/api/uploads/"+e.up.UID, atok, nil) + httpOK(t, w, 200, "status ok") + var body struct { + Received []int64 `json:"received"` + } + if err := jsonUnmarshal(w, &body); err != nil { + t.Fatal(err) + } + if len(body.Received) != 2 || body.Received[1] != 2 { + t.Fatalf("received want [0 2] got %v", body.Received) + } +} + +func TestUnit_UploadPart_BadIndex(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPut, "/api/uploads/"+e.up.UID+"/parts/abc", atok, nil) + httpOK(t, w, 400, "part bad index") +} + +func TestUnit_UploadPart_TooBig(t *testing.T) { + e := newTestEnv(t) + e.up.PutErr = upload.ErrPartTooBig + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPut, "/api/uploads/"+e.up.UID+"/parts/0", atok, nil) + httpOK(t, w, 413, "part too big") +} + +func TestUnit_UploadComplete_Incomplete(t *testing.T) { + e := newTestEnv(t) + e.seedLib(t, "up") // fake LibraryID=1 与 seed 的第一条对齐 + e.up.CompleteErr = ports.ErrIncomplete + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/uploads/"+e.up.UID+"/complete", atok, nil) + httpOK(t, w, 400, "complete incomplete") +} + +func TestUnit_UploadComplete_OK(t *testing.T) { + e := newTestEnv(t) + e.seedLib(t, "up") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/uploads/"+e.up.UID+"/complete", atok, nil) + httpOK(t, w, 202, "complete ok") + body := jsonBody(t, w) + if body["path"] != e.up.RelPath { + t.Fatalf("path want %q got %v", e.up.RelPath, body["path"]) + } +} + +func TestUnit_UploadComplete_LibGone(t *testing.T) { + e := newTestEnv(t) // LibraryID=1 但库里没有 id=1 → GetLibrary ErrNoRows → 404? dbErr → 500 + e.up.LibID = 42 + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/uploads/"+e.up.UID+"/complete", atok, nil) + // dbErr 把 pgx.ErrNoRows 归为 500(非连接类),契约与真库一致 + if w.Code != 500 { + t.Fatalf("lib gone want 500 got %d body=%s", w.Code, w.Body.String()) + } +} diff --git a/backend/cmd/webui/handlers/progress.go b/backend/cmd/webui/handlers/progress.go index 6c130f4..9a94565 100644 --- a/backend/cmd/webui/handlers/progress.go +++ b/backend/cmd/webui/handlers/progress.go @@ -32,7 +32,7 @@ func (h *H) PutProgress(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "locator must be valid json") return } - if e := h.st.UpsertProgress(c, uid(c), b.LibraryID, b.Path, req.Locator, req.Percent); e != nil { + if e := h.progress.UpsertProgress(c, uid(c), b.LibraryID, b.Path, req.Locator, req.Percent); e != nil { dbErr(c, e) return } @@ -40,7 +40,7 @@ func (h *H) PutProgress(c *gin.Context) { } func (h *H) ListProgress(c *gin.Context) { - rows, e := h.st.ListProgress(c, uid(c)) + rows, e := h.progress.ListProgress(c, uid(c)) if e != nil { dbErr(c, e) return diff --git a/backend/cmd/webui/handlers/unit_harness_test.go b/backend/cmd/webui/handlers/unit_harness_test.go new file mode 100644 index 0000000..71db936 --- /dev/null +++ b/backend/cmd/webui/handlers/unit_harness_test.go @@ -0,0 +1,142 @@ +package handlers_test + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gin-gonic/gin" + + "booklib/cmd/webui/api" + "booklib/internal/auth" + "booklib/internal/config" + "booklib/internal/ports/portsfake" +) + +// testEnv 是纯 fake 装配(Task 27):无 PG/Redis。请求走 api.NewRouter 组装的 +// 真实 engine,因此认证/鉴权中间件、路由匹配与 gin 的 header 落盘行为都被覆盖; +// 集成测试(auth_test.go 的 setupAPI)另走真库,两层互补。 +type testEnv struct { + cfg *config.Config + router *gin.Engine + users *portsfake.Users + libs *portsfake.Libraries + books *portsfake.Books + progress *portsfake.Progress + bookmarks *portsfake.Bookmarks + rl *portsfake.RateLimiter + sc *portsfake.Scanner + med *portsfake.Media + up *portsfake.Uploads + booksDir string +} + +func newTestEnv(t *testing.T) *testEnv { + t.Helper() + gin.SetMode(gin.TestMode) + resolved, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), UploadMaxMB: 1, + ScanInterval: time.Minute, BooksDir: resolved, CacheDir: t.TempDir(), + TrustedProxies: []string{"172.16.0.0/12"}} + users := portsfake.NewUsers() + libs := portsfake.NewLibraries() + books := portsfake.NewBooks() + progress := portsfake.NewProgress(libs, books) + e := &testEnv{ + cfg: cfg, users: users, libs: libs, books: books, + progress: progress, bookmarks: portsfake.NewBookmarks(), + rl: portsfake.NewRateLimiter(), sc: portsfake.NewScanner(), + med: portsfake.NewMedia(), up: portsfake.NewUploads(), + booksDir: resolved, + } + e.router = api.NewRouter(cfg, users, libs, books, progress, e.bookmarks, + e.rl, e.sc, e.med, e.up) + return e +} + +// token 签发一个带角色的 JWT(uid 固定 1,与 fake 里 seed 的用户对应)。 +func (e *testEnv) token(t *testing.T, role string, uid int64) string { + t.Helper() + tok, err := auth.Sign(e.cfg.JWTSecret, uid, role) + if err != nil { + t.Fatal(err) + } + return tok +} + +// do 以 Bearer token 走完整 engine;body 非 nil 时按 JSON 发送。 +func (e *testEnv) do(t *testing.T, method, path, tok string, body any) *httptest.ResponseRecorder { + t.Helper() + var reader *bytes.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + reader = bytes.NewReader(b) + } else { + reader = bytes.NewReader(nil) + } + req := httptest.NewRequest(method, path, reader) + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + w := httptest.NewRecorder() + e.router.ServeHTTP(w, req) + return w +} + +// jsonBody 解析响应体为 map。 +func jsonBody(t *testing.T, w *httptest.ResponseRecorder) map[string]any { + t.Helper() + var out map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("unmarshal %q: %v", w.Body.String(), err) + } + return out +} + +// jsonUnmarshal 解析响应体到任意目标(slice/struct)。 +func jsonUnmarshal(w *httptest.ResponseRecorder, dst any) error { + return json.Unmarshal(w.Body.Bytes(), dst) +} + +// errCode 取 error.code 字段(错误响应契约的核心)。 +func errCode(t *testing.T, w *httptest.ResponseRecorder) string { + t.Helper() + body := jsonBody(t, w) + e, _ := body["error"].(map[string]any) + if e == nil { + t.Fatalf("no error object in %q", w.Body.String()) + } + return fmt.Sprint(e["code"]) +} + +// seedLib 建一个 root 落在 BooksDir 内的库(fake 行 + 真实目录),返回 id 与 root。 +func (e *testEnv) seedLib(t *testing.T, name string) (int64, string) { + t.Helper() + root := filepath.Join(e.booksDir, name) + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + return e.libs.Seed(name, root), root +} + +// httpOK 断言状态码,失败时带上响应体便于定位。 +func httpOK(t *testing.T, w *httptest.ResponseRecorder, want int, what string) { + t.Helper() + if w.Code != want { + t.Fatalf("%s: want %d got %d body=%q", what, want, w.Code, w.Body.String()) + } +} diff --git a/backend/cmd/webui/handlers/uploads.go b/backend/cmd/webui/handlers/uploads.go index e0506c3..4627c90 100644 --- a/backend/cmd/webui/handlers/uploads.go +++ b/backend/cmd/webui/handlers/uploads.go @@ -1,89 +1,29 @@ package handlers import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" - "fmt" - "io" "net/http" "os" - "path/filepath" - "sort" "strconv" - "strings" - "time" "github.com/gin-gonic/gin" - "booklib/internal/bookfile" + "booklib/internal/ports" + "booklib/internal/upload" ) -// 分片上传:init(指纹→确定性 uploadId,天然支持续传)→ PUT parts → complete 拼接原子落盘。 -// 会话即 /.uploads//(meta.json + parts/N),无独立状态存储;24h 未 complete opportunistic 清扫。 +// 分片上传:HTTP 层只做参数绑定与错误映射,域逻辑(指纹续传/分片落盘/拼接/清扫) +// 全部在 internal/upload;会话清扫由 scanner ticker 接管(B16),不在请求路径。 -const ( - maxChunkBytes = 32 << 20 - defaultChunk = 8 << 20 - uploadSessTTL = 24 * time.Hour - uploadSessionIn = ".uploads" -) - -type uploadMeta struct { - Name string `json:"name"` - Size int64 `json:"size"` - ChunkSize int64 `json:"chunkSize"` - LibraryID int64 `json:"libraryId"` -} - -func validUploadID(s string) bool { - if len(s) != 32 { - return false - } - for _, r := range s { - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) { - return false - } - } - return true -} - -func uploadIDFor(libID int64, name string, size, chunk int64) string { - h := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%d|%d", libID, name, size, chunk))) - return hex.EncodeToString(h[:16]) -} - -func (h *H) uploadDir(uid string) string { - return filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn, uid) -} - -func chunkRange(m uploadMeta, i int64) (int64, int64) { - lo := i * m.ChunkSize - hi := min(lo+m.ChunkSize, m.Size) - return lo, hi -} - -func (h *H) numParts(m uploadMeta) int64 { - return (m.Size + m.ChunkSize - 1) / m.ChunkSize -} - -// sweepUploads 删除过期会话目录;尽力而为,失败不影响主流程 -func (h *H) sweepUploads() { - base := filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn) - es, e := os.ReadDir(base) - if e != nil { - return - } - for _, en := range es { - if fi, e := en.Info(); e == nil && time.Since(fi.ModTime()) > uploadSessTTL { - os.RemoveAll(filepath.Join(base, en.Name())) - } - } -} +// maxChunkBytes 与 upload 包内常量同值,作为 PutPart 的防御性读取上限。 +const maxChunkBytes = 32 << 20 func (h *H) UploadInit(c *gin.Context) { - lib, ok := h.getLibrary(c) + id, ok := idParam(c) + if !ok { + return + } + lib, ok := h.getLib(c, id) if !ok { return } @@ -99,146 +39,47 @@ func (h *H) UploadInit(c *gin.Context) { err(c, http.StatusBadRequest, "bad_request", "name, size required") return } - name := bookfile.SafeName(req.Name) - if name == "" { - err(c, http.StatusBadRequest, "bad_request", "bad name") - return - } - if bookfile.FormatFromExt(name) == "" { - err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md") - return - } - if req.Size <= 0 { - err(c, http.StatusBadRequest, "bad_request", "bad size") - return - } - if req.Size > h.cfg.UploadMaxMB<<20 { - err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB") - return - } - if req.ChunkSize == 0 { - req.ChunkSize = defaultChunk - } - if req.ChunkSize > maxChunkBytes { - err(c, http.StatusBadRequest, "bad_request", "chunkSize must be <= 33554432") - return - } - uid := uploadIDFor(lib.ID, name, req.Size, req.ChunkSize) - dir := h.uploadDir(uid) - meta := uploadMeta{Name: name, Size: req.Size, ChunkSize: req.ChunkSize, LibraryID: lib.ID} - if b, e := os.ReadFile(filepath.Join(dir, "meta.json")); e == nil { - var old uploadMeta - if json.Unmarshal(b, &old) == nil && old == meta { // 同指纹 → 复用会话(续传) - c.JSON(http.StatusOK, gin.H{"uploadId": uid}) - return - } - os.RemoveAll(dir) // 指纹撞上但内容不同 → 从头再来 - } - h.sweepUploads() - if e := os.MkdirAll(filepath.Join(dir, "parts"), 0o755); e != nil { - err(c, http.StatusInternalServerError, "internal", "create session") - return - } - b, _ := json.Marshal(meta) - if e := os.WriteFile(filepath.Join(dir, "meta.json"), b, 0o644); e != nil { - err(c, http.StatusInternalServerError, "internal", "write meta") + uid, e := h.up.Init(c, lib.ID, req.Name, req.Size, req.ChunkSize) + if e != nil { + h.mapUploadErr(c, e) return } c.JSON(http.StatusOK, gin.H{"uploadId": uid}) } -// loadMeta 校验 uid 与路径,404/400 已回复 -func (h *H) loadMeta(c *gin.Context) (uploadMeta, string, bool) { - uid := c.Param("uid") - if !validUploadID(uid) { - err(c, http.StatusBadRequest, "bad_request", "bad upload id") - return uploadMeta{}, "", false - } - dir := h.uploadDir(uid) - var m uploadMeta - b, e := os.ReadFile(filepath.Join(dir, "meta.json")) - if e != nil { - err(c, http.StatusNotFound, "not_found", "no such upload") - return uploadMeta{}, "", false - } - if json.Unmarshal(b, &m) != nil { - err(c, http.StatusInternalServerError, "internal", "corrupt session") - return uploadMeta{}, "", false - } - return m, dir, true -} - func (h *H) UploadStatus(c *gin.Context) { - _, dir, ok := h.loadMeta(c) - if !ok { + recv, e := h.up.Status(c, c.Param("uid")) + if e != nil { + h.mapUploadErr(c, e) return } - recv := []int64{} - es, e := os.ReadDir(filepath.Join(dir, "parts")) - if e == nil { - for _, en := range es { - if i, e := strconv.ParseInt(en.Name(), 10, 64); e == nil { - recv = append(recv, i) - } - } - } - sort.Slice(recv, func(i, j int) bool { return recv[i] < recv[j] }) c.JSON(http.StatusOK, gin.H{"received": recv}) } func (h *H) UploadPart(c *gin.Context) { - m, dir, ok := h.loadMeta(c) - if !ok { - return - } + uid := c.Param("uid") idx, e := strconv.ParseInt(c.Param("index"), 10, 64) - if e != nil || idx < 0 || idx >= h.numParts(m) { + if e != nil { err(c, http.StatusBadRequest, "bad_request", "bad part index") return } - lo, hi := chunkRange(m, idx) - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, hi-lo) - p := filepath.Join(dir, "parts", strconv.FormatInt(idx, 10)) - f, e := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + // maxSize 防御性上限:分片声明大小由会话 meta 决定,这里再垫一层 32MB 全局上限 + e = h.up.PutPart(c, uid, idx, c.Request.Body, maxChunkBytes) if e != nil { - err(c, http.StatusInternalServerError, "internal", "create part") - return - } - n, e := io.Copy(f, c.Request.Body) - f.Close() - if e != nil || n != hi-lo { - os.Remove(p) - var mbe *http.MaxBytesError - code, msg := "too_large", "part size mismatch" - if errors.As(e, &mbe) { - msg = "part exceeds declared size" - } - err(c, http.StatusRequestEntityTooLarge, code, msg) + h.mapUploadErr(c, e) return } c.JSON(http.StatusAccepted, gin.H{"accepted": true}) } func (h *H) UploadComplete(c *gin.Context) { - m, dir, ok := h.loadMeta(c) - if !ok { + uid := c.Param("uid") + libID, e := h.up.LibraryID(c, uid) + if e != nil { + h.mapUploadErr(c, e) return } - var total int64 - for i := int64(0); i < h.numParts(m); i++ { - lo, hi := chunkRange(m, i) - fi, e := os.Stat(filepath.Join(dir, "parts", strconv.FormatInt(i, 10))) - if e != nil || fi.Size() != hi-lo { - err(c, http.StatusBadRequest, "bad_request", "upload incomplete; missing or corrupt parts, re-upload them") - return - } - total += fi.Size() - } - if total != m.Size { - err(c, http.StatusBadRequest, "bad_request", "total size mismatch") - return - } - lib, e := h.st.GetLibrary(c, m.LibraryID) + lib, e := h.libs.GetLibrary(c, libID) if e != nil { dbErr(c, e) return @@ -247,39 +88,53 @@ func (h *H) UploadComplete(c *gin.Context) { if !ok { return } - dst, e := h.uniquePath(root, m.Name) + rel, e := h.up.Complete(c, uid, root) if e != nil { - err(c, http.StatusForbidden, "forbidden", e.Error()) + h.mapUploadErr(c, e) return } - tmp := filepath.Join(dir, "assembled") - out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - if e != nil { - err(c, http.StatusInternalServerError, "internal", "create tmp") - return - } - for i := int64(0); i < h.numParts(m); i++ { - pf, e := os.Open(filepath.Join(dir, "parts", strconv.FormatInt(i, 10))) - if e != nil { - out.Close() - err(c, http.StatusInternalServerError, "internal", "open part") - return - } - if _, e := io.Copy(out, pf); e != nil { - pf.Close() - out.Close() - os.Remove(tmp) - err(c, http.StatusInternalServerError, "internal", "assemble") + c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": rel}) +} + +// mapUploadErr 把 upload 包的 sentinel 错误映射为原契约的 status/code/message。 +func (h *H) mapUploadErr(c *gin.Context, e error) { + switch { + case errors.Is(e, ports.ErrTooLarge): + err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB") + case errors.Is(e, upload.ErrBadName): + err(c, http.StatusBadRequest, "bad_request", "bad name") + case errors.Is(e, upload.ErrBadFormat): + err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md") + case errors.Is(e, upload.ErrBadSize): + err(c, http.StatusBadRequest, "bad_request", "bad size") + case errors.Is(e, upload.ErrBadChunk): + err(c, http.StatusBadRequest, "bad_request", "chunkSize must be <= 33554432") + case errors.Is(e, upload.ErrBadUploadID): + err(c, http.StatusBadRequest, "bad_request", "bad upload id") + case errors.Is(e, upload.ErrBadIndex): + err(c, http.StatusBadRequest, "bad_request", "bad part index") + case errors.Is(e, upload.ErrPartTooBig): + err(c, http.StatusRequestEntityTooLarge, "too_large", "part exceeds declared size") + case errors.Is(e, upload.ErrPartSizeMismatch): + err(c, http.StatusRequestEntityTooLarge, "too_large", "part size mismatch") + case errors.Is(e, ports.ErrNotFound): + err(c, http.StatusNotFound, "not_found", "no such upload") + case errors.Is(e, upload.ErrCorrupt): + err(c, http.StatusInternalServerError, "internal", "corrupt session") + case errors.Is(e, ports.ErrIncomplete): + err(c, http.StatusBadRequest, "bad_request", "upload incomplete; missing or corrupt parts, re-upload them") + case errors.Is(e, ports.ErrSizeMismatch): + err(c, http.StatusBadRequest, "bad_request", "total size mismatch") + case errors.Is(e, os.ErrInvalid), errors.Is(e, os.ErrExist): + err(c, http.StatusForbidden, "forbidden", e.Error()) + case errors.Is(e, os.ErrPermission), errors.Is(e, os.ErrClosed): + err(c, http.StatusInternalServerError, "internal", "io error") + default: + var oe *upload.OpError + if errors.As(e, &oe) { + err(c, http.StatusInternalServerError, "internal", oe.Op) return } - pf.Close() - } - out.Close() - if e := os.Rename(tmp, dst); e != nil { // 原子落盘,scanner 自动收编 - os.Remove(tmp) - err(c, http.StatusInternalServerError, "internal", "rename") - return + err(c, http.StatusInternalServerError, "internal", "upload failed") } - os.RemoveAll(dir) - c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": strings.TrimPrefix(dst, root+string(os.PathSeparator))}) } diff --git a/backend/cmd/webui/handlers/users.go b/backend/cmd/webui/handlers/users.go index 821f7f5..201c281 100644 --- a/backend/cmd/webui/handlers/users.go +++ b/backend/cmd/webui/handlers/users.go @@ -3,23 +3,18 @@ package handlers import ( "errors" "net/http" - "strconv" "time" "github.com/gin-gonic/gin" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "booklib/internal/auth" + "booklib/internal/ports" + "booklib/internal/store" ) -func isUnique(e error) bool { - var pgErr *pgconn.PgError - return errors.As(e, &pgErr) && pgErr.Code == "23505" -} - func (h *H) ListUsers(c *gin.Context) { - users, e := h.st.ListUsers(c) + users, e := h.users.ListUsers(c) if e != nil { dbErr(c, e) return @@ -51,9 +46,9 @@ func (h *H) CreateUser(c *gin.Context) { err(c, http.StatusInternalServerError, "internal", "hash") return } - id, e := h.st.CreateUser(c, req.Username, hp, req.Role) + id, e := h.users.CreateUser(c, req.Username, hp, req.Role) if e != nil { - if isUnique(e) { + if ports.IsUniqueViolation(e) { err(c, http.StatusConflict, "exists", "username taken") return } @@ -64,32 +59,24 @@ func (h *H) CreateUser(c *gin.Context) { } func (h *H) DeleteUser(c *gin.Context) { - id, e := strconv.ParseInt(c.Param("id"), 10, 64) - if e != nil { - err(c, http.StatusBadRequest, "bad_request", "bad id") + id, ok := idParam(c) + if !ok { return } if id == uid(c) { err(c, http.StatusBadRequest, "bad_request", "cannot delete yourself") return } - target, e := h.st.GetUserByID(c, id) - if e != nil { + // B5: transactional last-admin check eliminates TOCTOU race. + if e := h.users.DeleteUser(c, id); e != nil { if errors.Is(e, pgx.ErrNoRows) { err(c, http.StatusNotFound, "not_found", "no such user") return } - dbErr(c, e) - return - } - if target.Role == "admin" { - n, _ := h.st.CountAdmins(c) // 防删光最后一个 admin - if n <= 1 { + if errors.Is(e, store.ErrLastAdmin) { err(c, http.StatusBadRequest, "bad_request", "cannot delete the last admin") return } - } - if e := h.st.DeleteUser(c, id); e != nil { dbErr(c, e) return } diff --git a/backend/cmd/webui/handlers/users_unit_test.go b/backend/cmd/webui/handlers/users_unit_test.go new file mode 100644 index 0000000..240dd8b --- /dev/null +++ b/backend/cmd/webui/handlers/users_unit_test.go @@ -0,0 +1,154 @@ +package handlers_test + +import ( + "context" + "errors" + "net/http" + "strconv" + "testing" + + "github.com/jackc/pgx/v5" + + "booklib/internal/auth" +) + +// ---------- users: admin-only CRUD branches (Task 27, portsfake) ---------- + +func TestUnit_ListUsers(t *testing.T) { + e := newTestEnv(t) + hash, _ := auth.HashPassword("password1234") + e.users.Seed("admin", hash, "admin") + e.users.Seed("member1", hash, "member") + atok := e.token(t, "admin", 1) + + w := e.do(t, http.MethodGet, "/api/users", atok, nil) + httpOK(t, w, 200, "list users") + var users []map[string]any + if err := jsonUnmarshal(w, &users); err != nil { + t.Fatal(err) + } + if len(users) != 2 { + t.Fatalf("want 2 users got %d", len(users)) + } +} + +func TestUnit_ListUsers_MemberForbidden(t *testing.T) { + e := newTestEnv(t) + mtok := e.token(t, "member", 2) + w := e.do(t, http.MethodGet, "/api/users", mtok, nil) + httpOK(t, w, 403, "member list users") +} + +func TestUnit_CreateUser_RoleValidation(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + for _, role := range []string{"superadmin", "", "Member"} { + w := e.do(t, http.MethodPost, "/api/users", atok, + map[string]string{"Username": "u", "Password": "password1234", "Role": role}) + httpOK(t, w, 400, "create user role="+role) + } +} + +func TestUnit_CreateUser_ShortPassword(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/users", atok, + map[string]string{"Username": "u", "Password": "short", "Role": "member"}) + httpOK(t, w, 400, "short password") +} + +func TestUnit_CreateUser_DuplicateName(t *testing.T) { + e := newTestEnv(t) + hash, _ := auth.HashPassword("password1234") + e.users.Seed("existing", hash, "member") + atok := e.token(t, "admin", 1) + + w := e.do(t, http.MethodPost, "/api/users", atok, + map[string]string{"Username": "existing", "Password": "password1234", "Role": "member"}) + httpOK(t, w, 409, "duplicate name") + if code := errCode(t, w); code != "exists" { + t.Fatalf("error code want 'exists' got %q", code) + } +} + +func TestUnit_CreateUser_OK(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodPost, "/api/users", atok, + map[string]string{"Username": "newbie", "Password": "password1234", "Role": "member"}) + httpOK(t, w, 201, "create user") + body := jsonBody(t, w) + if body["username"] != "newbie" || body["role"] != "member" { + t.Fatalf("unexpected body %v", body) + } +} + +func TestUnit_DeleteUser_BadID(t *testing.T) { + e := newTestEnv(t) + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodDelete, "/api/users/notanum", atok, nil) + httpOK(t, w, 400, "bad id") +} + +func TestUnit_DeleteUser_Self(t *testing.T) { + e := newTestEnv(t) + uid := e.users.Seed("self", "hash", "admin") + atok := e.token(t, "admin", uid) // 自己删自己 + w := e.do(t, http.MethodDelete, "/api/users/"+strconv.FormatInt(uid, 10), atok, nil) + httpOK(t, w, 400, "delete self") +} + +func TestUnit_DeleteUser_LastAdmin(t *testing.T) { + e := newTestEnv(t) + uid := e.users.Seed("lastadmin", "hash", "admin") + atok := e.token(t, "admin", 999) // 另一个(不存在的)操作者 + w := e.do(t, http.MethodDelete, "/api/users/"+strconv.FormatInt(uid, 10), atok, nil) + httpOK(t, w, 400, "last admin") +} + +func TestUnit_DeleteUser_NotFound(t *testing.T) { + e := newTestEnv(t) + e.users.Seed("other", "hash", "admin") + atok := e.token(t, "admin", 1) + w := e.do(t, http.MethodDelete, "/api/users/99999", atok, nil) + httpOK(t, w, 404, "missing user") +} + +func TestUnit_DeleteUser_OK(t *testing.T) { + e := newTestEnv(t) + e.users.Seed("admin", "hash", "admin") // 保住 last-admin 保护不触发 + target := e.users.Seed("todelete", "hash", "member") + atok := e.token(t, "admin", 1) + + w := e.do(t, http.MethodDelete, "/api/users/"+strconv.FormatInt(target, 10), atok, nil) + httpOK(t, w, 204, "delete user") + if _, err := e.users.GetUserByID(context.Background(), target); !errors.Is(err, pgx.ErrNoRows) { + t.Fatal("user should be gone") + } +} + +func TestUnit_Me_OK(t *testing.T) { + e := newTestEnv(t) + uid := e.users.Seed("me", "hash", "member") + mtok := e.token(t, "member", uid) + + w := e.do(t, http.MethodGet, "/api/auth/me", mtok, nil) + httpOK(t, w, 200, "me") + body := jsonBody(t, w) + if body["username"] != "me" { + t.Fatalf("username want 'me' got %v", body["username"]) + } +} + +func TestUnit_Me_UserGone(t *testing.T) { + e := newTestEnv(t) + mtok := e.token(t, "member", 99999) + w := e.do(t, http.MethodGet, "/api/auth/me", mtok, nil) + httpOK(t, w, 401, "me after user gone") +} + +func TestUnit_Me_NoToken(t *testing.T) { + e := newTestEnv(t) + w := e.do(t, http.MethodGet, "/api/auth/me", "", nil) + httpOK(t, w, 401, "me without token") +} diff --git a/backend/cmd/webui/main.go b/backend/cmd/webui/main.go index f487e59..e6a8dca 100644 --- a/backend/cmd/webui/main.go +++ b/backend/cmd/webui/main.go @@ -12,10 +12,12 @@ import ( "booklib/cmd/webui/api" "booklib/internal/config" "booklib/internal/db" + "booklib/internal/media" "booklib/internal/redispkg" "booklib/internal/scanner" "booklib/internal/seed" "booklib/internal/store" + "booklib/internal/upload" ) func main() { @@ -39,18 +41,31 @@ func main() { log.Fatalf("seed: %v", err) } rdb := redispkg.New(cfg.RedisURL) - sc := scanner.New(st, cfg, rdb) + med := media.New(cfg, rdb) + up := upload.New(cfg.BooksDir, cfg.UploadMaxMB) + sc := scanner.New(st, cfg, rdb, up) // B16: sweep rides the scan ticker go sc.Run(ctx) - srv := &http.Server{Addr: cfg.Addr, Handler: api.NewRouter(cfg, st, rdb, sc), + serveErr := make(chan error, 1) + // *store.Store 同时满足 5 个 store 接口;*redispkg.R 满足 RateLimiter。 + srv := &http.Server{Addr: cfg.Addr, Handler: api.NewRouter(cfg, st, st, st, st, st, rdb, sc, med, up), ReadHeaderTimeout: 10 * time.Second} go func() { log.Printf("listening on %s", cfg.Addr) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Fatalf("serve: %v", err) + serveErr <- err } + close(serveErr) }() - <-ctx.Done() + + // Wait for signal OR serve error. + select { + case <-ctx.Done(): + case err := <-serveErr: + if err != nil { + log.Printf("serve: %v", err) + } + } // stop() 先取消 ctx → scanner 循环退出;再等 HTTP 收尾。 // 在途 ScanLibraryByID(WithoutCancel)不受 ctx 控制,靠 redis 锁 TTL 兜底(已文档化的上限)。 stop() diff --git a/backend/internal/bookfile/reader.go b/backend/internal/bookfile/reader.go new file mode 100644 index 0000000..6bc9dcf --- /dev/null +++ b/backend/internal/bookfile/reader.go @@ -0,0 +1,37 @@ +package bookfile + +import ( + "os" + "path/filepath" + "strings" +) + +// OpenReaderAt opens a book file and returns a ReaderAt + size. +// Consolidates the 3 places that open a book file + stat + get ReaderAt. +func OpenReaderAt(root, rel string) (*os.File, int64, error) { + abs := filepath.Join(root, filepath.FromSlash(rel)) + f, err := os.Open(abs) + if err != nil { + return nil, 0, err + } + st, err := f.Stat() + if err != nil { + f.Close() + return nil, 0, err + } + return f, st.Size(), nil +} + +// Contains reports whether child is inside parent using EvalSymlinks semantics. +// Falls back to Clean if EvalSymlinks fails (e.g., path doesn't exist yet). +func Contains(parent, child string) bool { + p, err := filepath.EvalSymlinks(parent) + if err != nil { + p = filepath.Clean(parent) + } + c, err := filepath.EvalSymlinks(child) + if err != nil { + c = filepath.Clean(child) + } + return c == p || strings.HasPrefix(c, p+string(os.PathSeparator)) +} diff --git a/backend/internal/bookfile/reader_test.go b/backend/internal/bookfile/reader_test.go new file mode 100644 index 0000000..e962eab --- /dev/null +++ b/backend/internal/bookfile/reader_test.go @@ -0,0 +1,51 @@ +package bookfile + +import ( + "os" + "path/filepath" + "testing" +) + +func TestContains(t *testing.T) { + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if !Contains(dir, sub) { + t.Fatal("sub should be inside dir") + } + if Contains(sub, dir) { + t.Fatal("dir should not be inside sub") + } + if Contains(dir, "/completely/different") { + t.Fatal("unrelated path should not be inside dir") + } + // Same path. + if !Contains(dir, dir) { + t.Fatal("dir should contain itself") + } +} + +func TestOpenReaderAt(t *testing.T) { + dir := t.TempDir() + // Create a test file. + path := filepath.Join(dir, "test.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + f, size, err := OpenReaderAt(dir, "test.txt") + if err != nil { + t.Fatal(err) + } + defer f.Close() + if size != 5 { + t.Fatalf("size = %d, want 5", size) + } + + // Non-existent file. + _, _, err = OpenReaderAt(dir, "nope.txt") + if err == nil { + t.Fatal("expected error for non-existent file") + } +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 8e06393..394ac8c 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -2,11 +2,14 @@ package config import ( "fmt" + "log" "os" "path/filepath" "strconv" "strings" "time" + + "github.com/jackc/pgx/v5/pgxpool" ) type Config struct { @@ -52,6 +55,18 @@ func Load() (*Config, error) { if secret == "" { return nil, fmt.Errorf("JWT_SECRET required") } + // Validate required fields. + dbURL := env("DATABASE_URL", "") + if dbURL == "" { + return nil, fmt.Errorf("DATABASE_URL is required") + } + if _, perr := pgxpool.ParseConfig(dbURL); perr != nil { + return nil, fmt.Errorf("DATABASE_URL: %w", perr) + } + if env("REDIS_URL", "") == "" { + log.Printf("redis disabled: rate-limit/scan-lock/page-cache off") + } + // resolveDir: macOS 开发机上 /var、/tmp 是指向 /private 的软链, // 启动时解析一次才能让 root_path 前缀校验对上真实路径;目录不存在/出错则保留原值(Docker 路径不受影响) resolveDir := func(dir string) string { diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index c6fd5c0..1db8cf9 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -6,6 +6,22 @@ import ( "time" ) +func TestLoadDatabaseURLRequired(t *testing.T) { + t.Setenv("JWT_SECRET", "x") + t.Setenv("DATABASE_URL", "") + if _, err := Load(); err == nil { + t.Fatal("empty DATABASE_URL must fail") + } +} + +func TestLoadDatabaseURLMalformed(t *testing.T) { + t.Setenv("JWT_SECRET", "x") + t.Setenv("DATABASE_URL", "not a url") + if _, err := Load(); err == nil { + t.Fatal("malformed DATABASE_URL must fail") + } +} + func TestLoad(t *testing.T) { // 屏蔽外部 env,保证默认值断言自洽(Load 将空串视为未设置) t.Setenv("BOOKS_DIR", "") diff --git a/backend/internal/db/db.go b/backend/internal/db/db.go index ce3228d..d36cf80 100644 --- a/backend/internal/db/db.go +++ b/backend/internal/db/db.go @@ -2,14 +2,26 @@ package db import ( "context" - _ "embed" + "embed" "fmt" + "io/fs" + "log" + "regexp" + "sort" + "strings" "github.com/jackc/pgx/v5/pgxpool" ) -//go:embed schema.sql -var schema string +//go:embed migrations +var migrationsFS embed.FS + +// advisoryLockKey is a fixed int64 used with pg_advisory_lock to serialize +// migrations across --scale api=N replicas. Value is arbitrary but must be +// unique within the database (pick a project-specific constant). +const advisoryLockKey int64 = 0x424C4D49 // "BLMI" + +var migrationNameRe = regexp.MustCompile(`^\d{4}_[a-z0-9_]+\.sql$`) func Connect(ctx context.Context, url string) (*pgxpool.Pool, error) { cfg, err := pgxpool.ParseConfig(url) @@ -21,8 +33,116 @@ func Connect(ctx context.Context, url string) (*pgxpool.Pool, error) { } func Migrate(ctx context.Context, p *pgxpool.Pool) error { - if _, err := p.Exec(ctx, schema); err != nil { - return fmt.Errorf("migrate: %w", err) + // 1. Acquire advisory lock — serializes concurrent replicas. + if _, err := p.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockKey); err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + defer func() { + if _, err := p.Exec(ctx, "SELECT pg_advisory_unlock($1)", advisoryLockKey); err != nil { + log.Printf("advisory unlock: %v", err) + } + }() + + // 2. Create tracking table. + if _, err := p.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`); err != nil { + return fmt.Errorf("create schema_migrations: %w", err) + } + + // 3. Read embedded migration files, validate names. + entries, err := fs.ReadDir(migrationsFS, "migrations") + if err != nil { + return fmt.Errorf("read migrations dir: %w", err) + } + var files []string + for _, e := range entries { + name := e.Name() + if !migrationNameRe.MatchString(name) { + panic(fmt.Sprintf("invalid migration filename: %q (must match %s)", name, migrationNameRe)) + } + files = append(files, name) + } + sort.Strings(files) + + // 4. Baseline detection: if schema_migrations is empty but 'books' table exists, + // this is an existing database — mark 0001 as applied without re-running DDL. + var count int + if err := p.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&count); err != nil { + return fmt.Errorf("count migrations: %w", err) + } + if count == 0 { + var hasBooks bool + err := p.QueryRow(ctx, "SELECT to_regclass('books') IS NOT NULL").Scan(&hasBooks) + if err != nil { + return fmt.Errorf("check books table: %w", err) + } + if hasBooks && len(files) > 0 && strings.HasPrefix(files[0], "0001_") { + if _, err := p.Exec(ctx, + "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", + 1, files[0]); err != nil { + return fmt.Errorf("baseline insert: %w", err) + } + log.Printf("migration baseline: marked %s as applied (existing database)", files[0]) + files = files[1:] + } + } + + // 5. Build set of already-applied versions. + applied := map[int64]bool{} + rows, err := p.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return fmt.Errorf("list applied: %w", err) + } + defer rows.Close() + for rows.Next() { + var v int64 + if err := rows.Scan(&v); err != nil { + return fmt.Errorf("scan applied: %w", err) + } + applied[v] = true + } + if err := rows.Err(); err != nil { + return fmt.Errorf("rows applied: %w", err) + } + + // 6. Apply pending migrations in order, each in its own transaction. + for _, name := range files { + version := parseVersion(name) + if applied[version] { + continue + } + sql, err := fs.ReadFile(migrationsFS, "migrations/"+name) + if err != nil { + return fmt.Errorf("read %s: %w", name, err) + } + tx, err := p.Begin(ctx) + if err != nil { + return fmt.Errorf("begin %s: %w", name, err) + } + if _, err := tx.Exec(ctx, string(sql)); err != nil { + tx.Rollback(ctx) + return fmt.Errorf("exec %s: %w", name, err) + } + if _, err := tx.Exec(ctx, + "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", + version, name); err != nil { + tx.Rollback(ctx) + return fmt.Errorf("record %s: %w", name, err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit %s: %w", name, err) + } + log.Printf("migration applied: %s", name) } return nil } + +func parseVersion(name string) int64 { + parts := strings.SplitN(name, "_", 2) + var v int64 + fmt.Sscanf(parts[0], "%d", &v) + return v +} diff --git a/backend/internal/db/schema.sql b/backend/internal/db/migrations/0001_baseline.sql similarity index 100% rename from backend/internal/db/schema.sql rename to backend/internal/db/migrations/0001_baseline.sql diff --git a/backend/internal/media/media.go b/backend/internal/media/media.go new file mode 100644 index 0000000..55f0522 --- /dev/null +++ b/backend/internal/media/media.go @@ -0,0 +1,165 @@ +package media + +import ( + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "booklib/internal/bookfile" + "booklib/internal/config" + "booklib/internal/ports" + "booklib/internal/redispkg" +) + +// M is the media service — single source of truth for cache layout and content extraction. +type M struct { + cfg *config.Config + rdb *redispkg.R +} + +// compile-time proof that *M satisfies the consumer-side interface. +var _ ports.Media = (*M)(nil) + +func New(cfg *config.Config, rdb *redispkg.R) *M { + return &M{cfg: cfg, rdb: rdb} +} + +// CacheBuster returns a content hash for cache-busting URLs. +func (m *M) CacheBuster(size, modTS int64) string { + return bookfile.Hash(size, modTS) +} + +// EnsureCover extracts and caches the cover image for a book. +// Returns nil if the cover is already cached or the format doesn't support covers. +func (m *M) EnsureCover(ctx context.Context, bookID int64, format string, size, modTS int64, root, rel string) error { + var fn func(io.ReaderAt, int64) ([]byte, string, error) + switch format { + case "cbz": + fn = bookfile.CBZCover + case "epub": + fn = bookfile.EPUBCover + default: + return nil + } + f, fsize, err := bookfile.OpenReaderAt(root, rel) + if err != nil { + return err + } + defer f.Close() + img, ext, err := fn(f, fsize) + if err != nil { + return err + } + dir := bookfile.CoverDir(m.cfg.CacheDir, bookfile.DirKey(bookID, bookfile.Hash(size, modTS))) + return WriteAtomic(dir, "cover"+ext, img) +} + +// PageIndex returns the page list for a CBZ, using redis cache when available. +func (m *M) PageIndex(ctx context.Context, bookID int64, size, modTS int64, root, rel string) ([]string, error) { + hash := bookfile.Hash(size, modTS) + key := fmt.Sprintf("pagesidx2:%d:%s", bookID, hash) + if v, ok := m.rdb.Get(ctx, key); ok && v != "" { + return strings.Split(v, "\n"), nil + } + f, fsize, err := bookfile.OpenReaderAt(root, rel) + if err != nil { + return nil, err + } + defer f.Close() + idx, err := bookfile.PageIndex(f, fsize) + if err != nil { + return nil, err + } + if len(idx) > 0 { + m.rdb.Set(ctx, key, strings.Join(idx, "\n"), 7*24*time.Hour) + } + return idx, nil +} + +// ChaptersOf derives chapters from a page index by grouping pages by parent directory. +func (m *M) ChaptersOf(idx []string) []ports.Chapter { + type grp struct { + dir string + start int + } + var grps []grp + last := "\x00" + for i, n := range idx { + d := path.Dir(n) + if d == last { + continue + } + last = d + if d == "." { + continue + } + grps = append(grps, grp{d, i}) + } + if len(grps) < 2 { + return nil + } + titles := make(map[string]int) + out := make([]ports.Chapter, len(grps)) + for i, g := range grps { + out[i] = ports.Chapter{Title: path.Base(g.dir), Start: g.start} + titles[out[i].Title]++ + } + for i, g := range grps { + if titles[out[i].Title] > 1 { + out[i].Title = g.dir + } + } + return out +} + +// EnsurePage extracts a single page to the cache. Returns the cache file path. +func (m *M) EnsurePage(_ context.Context, bookID int64, size, modTS int64, root, rel string, n int, idx []string) (string, error) { + if n >= len(idx) { + return "", fmt.Errorf("page %d out of range", n) + } + ext := strings.ToLower(filepath.Ext(idx[n])) + dir := bookfile.PagesDir(m.cfg.CacheDir, bookfile.DirKey(bookID, bookfile.Hash(size, modTS))) + dst := filepath.Join(dir, strconv.Itoa(n)+ext) + if _, err := os.Stat(dst); err == nil { + return dst, nil // already cached + } + f, fsize, err := bookfile.OpenReaderAt(root, rel) + if err != nil { + return "", err + } + defer f.Close() + data, err := bookfile.ReadEntry(f, fsize, idx[n]) + if err != nil { + return "", err + } + if err := WriteAtomic(dir, strconv.Itoa(n)+ext, data); err != nil { + return "", err + } + return dst, nil +} + +// WriteAtomic writes data to dir/name via tmp+rename. B11: cleans tmp only on +// failure, so a crash mid-write never leaves a readable half-written file. +// It does not log — callers own the context (which book, which page). +func WriteAtomic(dir, name string, data []byte) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tmp := filepath.Join(dir, fmt.Sprintf("%s.tmp-%d", name, time.Now().UnixNano())) + if err := os.WriteFile(tmp, data, 0o644); err != nil { + os.Remove(tmp) + return err + } + dst := filepath.Join(dir, name) + if err := os.Rename(tmp, dst); err != nil { + os.Remove(tmp) + return err + } + return nil +} diff --git a/backend/internal/media/reserved.go b/backend/internal/media/reserved.go new file mode 100644 index 0000000..e8290ee --- /dev/null +++ b/backend/internal/media/reserved.go @@ -0,0 +1,16 @@ +package media + +import "strings" + +// reservedNames are library names that conflict with system directories. +// This is the single source of truth for reserved name validation (B8). +var reservedNames = map[string]bool{ + "cache": true, // CACHE_DIR + ".uploads": true, // upload session directory + ".trash": true, // potential future use +} + +// IsReservedName reports whether name conflicts with system directories. +func IsReservedName(name string) bool { + return reservedNames[strings.ToLower(name)] +} diff --git a/backend/internal/media/reserved_test.go b/backend/internal/media/reserved_test.go new file mode 100644 index 0000000..f26c1fd --- /dev/null +++ b/backend/internal/media/reserved_test.go @@ -0,0 +1,16 @@ +package media + +import "testing" + +func TestIsReservedName(t *testing.T) { + for _, n := range []string{"cache", "Cache", "CACHE", ".uploads", ".Uploads", ".trash"} { + if !IsReservedName(n) { + t.Errorf("IsReservedName(%q) = false, want true", n) + } + } + for _, n := range []string{"comics", "books", "my-library", "Cache1"} { + if IsReservedName(n) { + t.Errorf("IsReservedName(%q) = true, want false", n) + } + } +} diff --git a/backend/internal/ports/errors.go b/backend/internal/ports/errors.go new file mode 100644 index 0000000..1cc6dfa --- /dev/null +++ b/backend/internal/ports/errors.go @@ -0,0 +1,24 @@ +package ports + +import ( + "errors" + + "booklib/internal/store" +) + +// Re-export sentinel errors so handlers can use errors.Is via ports. +var ( + ErrLastAdmin = store.ErrLastAdmin + ErrUniqueViolation = store.ErrUniqueViolation +) + +// IsUniqueViolation consolidates the pg 23505 check into a single predicate. +func IsUniqueViolation(err error) bool { return store.IsUniqueViolation(err) } + +// Upload sentinel errors. +var ( + ErrTooLarge = errors.New("file too large") + ErrIncomplete = errors.New("upload incomplete") + ErrSizeMismatch = errors.New("size mismatch") + ErrNotFound = errors.New("not found") +) diff --git a/backend/internal/ports/ports.go b/backend/internal/ports/ports.go new file mode 100644 index 0000000..976a8e8 --- /dev/null +++ b/backend/internal/ports/ports.go @@ -0,0 +1,107 @@ +// Package ports defines consumer-side interfaces for the backend services. +// Handlers depend on these interfaces, not on concrete implementations. +// Main.go wires concrete types (*store.Store, *redispkg.R, etc.) that satisfy them. +// +// Value types (User, Book, etc.) live in internal/store and are referenced here. +package ports + +import ( + "context" + "io" + "time" + + "booklib/internal/store" +) + +// ---------- Store interfaces ---------- + +type UserStore interface { + CountUsers(ctx context.Context) (int, error) + CreateUser(ctx context.Context, username, hash, role string) (int64, error) + GetUserByName(ctx context.Context, username string) (store.User, error) + GetUserByID(ctx context.Context, id int64) (store.User, error) + ListUsers(ctx context.Context) ([]store.User, error) + DeleteUser(ctx context.Context, id int64) error +} + +type LibraryStore interface { + CreateLibrary(ctx context.Context, name, root string) (int64, error) + ListLibraries(ctx context.Context) ([]store.Library, error) + GetLibrary(ctx context.Context, id int64) (store.Library, error) +} + +type BookStore interface { + InsertBook(ctx context.Context, libID int64, path, title, format string, size, modTS int64, pageCount int) (int64, error) + GetBook(ctx context.Context, id int64) (store.Book, error) + ListBookMeta(ctx context.Context, libID int64) (map[string]store.BookMeta, error) + UpdateBookFile(ctx context.Context, id, size, modTS int64, pageCount int) error + DeleteBookByPath(ctx context.Context, libID int64, path string) error + DeleteBook(ctx context.Context, id int64) error + SetBookState(ctx context.Context, id int64, state, msg string) error + ListBooks(ctx context.Context, libID int64, q, prefix string, userID int64) ([]store.BookView, error) + BookHashes(ctx context.Context) (map[int64][2]int64, error) +} + +type ProgressStore interface { + UpsertProgress(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64) error + GetProgress(ctx context.Context, userID, libID int64, bookPath string) (store.Progress, error) + ListProgress(ctx context.Context, userID int64) ([]store.Progress, error) +} + +type BookmarkStore interface { + InsertBookmark(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64, note string) (int64, error) + ListBookmarks(ctx context.Context, userID, libID int64, bookPath string) ([]store.Bookmark, error) + UpdateBookmarkNote(ctx context.Context, userID, id int64, note string) (bool, error) + DeleteBookmark(ctx context.Context, userID, id int64) (bool, error) +} + +// ---------- Redis interfaces ---------- + +type PageCache interface { + Get(ctx context.Context, key string) (string, bool) + Set(ctx context.Context, key, val string, ttl time.Duration) +} + +type RateLimiter interface { + IncrWindow(ctx context.Context, key string, ttl time.Duration) int +} + +type ScanLocker interface { + ScanLock(ctx context.Context, key string, ttl time.Duration) (func(), bool) +} + +// ---------- Service interfaces ---------- + +type Scanner interface { + ScanLibraryByID(ctx context.Context, id int64) +} + +type UploadSessions interface { + Init(ctx context.Context, libID int64, name string, size, chunkSize int64) (string, error) + // LibraryID returns the target library recorded in the session, so the + // handler can resolve+validate the library root before Complete. + LibraryID(ctx context.Context, uploadID string) (int64, error) + Status(ctx context.Context, uploadID string) ([]int64, error) + PutPart(ctx context.Context, uploadID string, index int64, body io.Reader, maxSize int64) error + Complete(ctx context.Context, uploadID string, root string) (string, error) + Sweep(ctx context.Context) error + UniquePath(root, name string) (string, error) +} + +type Media interface { + // EnsureCover extracts+caches the cover if not already cached. Returns nil + // for formats without cover support. + EnsureCover(ctx context.Context, bookID int64, format string, size, modTS int64, root, rel string) error + // EnsurePage extracts page n (from a pre-fetched idx) to the cache and + // returns the cache file path. + EnsurePage(ctx context.Context, bookID int64, size, modTS int64, root, rel string, n int, idx []string) (string, error) + ChaptersOf(idx []string) []Chapter + PageIndex(ctx context.Context, bookID int64, size, modTS int64, root, rel string) ([]string, error) + CacheBuster(size, modTS int64) string +} + +// Chapter represents a CBZ chapter derived from the archive's folder structure. +type Chapter struct { + Title string `json:"title"` + Start int `json:"start"` +} diff --git a/backend/internal/ports/portsfake/services.go b/backend/internal/ports/portsfake/services.go new file mode 100644 index 0000000..8b01bab --- /dev/null +++ b/backend/internal/ports/portsfake/services.go @@ -0,0 +1,198 @@ +package portsfake + +import ( + "context" + "fmt" + "io" + "sync" + "time" + + "booklib/internal/ports" +) + +// ---------- RateLimiter ---------- + +// RateLimiter counts IncrWindow calls per key and returns a programmable +// window value. Default: always allow (return 1). +type RateLimiter struct { + mu sync.Mutex + calls map[string]int + Result int // value returned by IncrWindow; 0 means "call count" + Hook func(key string, n int) int // optional override +} + +func NewRateLimiter() *RateLimiter { return &RateLimiter{calls: map[string]int{}} } + +func (r *RateLimiter) IncrWindow(_ context.Context, key string, _ time.Duration) int { + r.mu.Lock() + r.calls[key]++ + n := r.calls[key] + r.mu.Unlock() + if r.Hook != nil { + return r.Hook(key, n) + } + if r.Result > 0 { + return r.Result + } + return n +} + +// Calls reports how many times IncrWindow was invoked for key. +func (r *RateLimiter) Calls(key string) int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[key] +} + +// ---------- Scanner ---------- + +// Scanner records ScanLibraryByID calls. The handler invokes it in a goroutine, +// so reads go through the mutex; WaitForScan blocks until at least n calls +// landed (or the timeout expires) to keep tests deterministic. +type Scanner struct { + mu sync.Mutex + seen []int64 +} + +func NewScanner() *Scanner { return &Scanner{} } + +func (s *Scanner) ScanLibraryByID(_ context.Context, id int64) { + s.mu.Lock() + s.seen = append(s.seen, id) + s.mu.Unlock() +} + +// Seen returns the library ids scanned so far, in call order. +func (s *Scanner) Seen() []int64 { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]int64, len(s.seen)) + copy(out, s.seen) + return out +} + +// WaitForScan blocks until len(Seen()) >= n or timeout elapses; reports success. +func (s *Scanner) WaitForScan(n int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if len(s.Seen()) >= n { + return true + } + time.Sleep(time.Millisecond) + } + return len(s.Seen()) >= n +} + +// ---------- Media ---------- + +// Media is a programmable stand-in for the media service. Defaults answer +// successfully with the configured fixture data; individual hooks let a test +// force one error branch without touching the others. +type Media struct { + Pages []string // PageIndex result + Chapters []ports.Chapter // ChaptersOf result + CoverErr error // EnsureCover result + IndexErr error // PageIndex result + PagePath string // EnsurePage result path (test writes the file first) + PageErr error // EnsurePage result + + EnsureCoverCalls int + EnsurePageCalls int +} + +func NewMedia() *Media { return &Media{} } + +func (m *Media) EnsureCover(_ context.Context, _ int64, _ string, _, _ int64, _, _ string) error { + m.EnsureCoverCalls++ + return m.CoverErr +} + +func (m *Media) EnsurePage(_ context.Context, _ int64, _, _ int64, _, _ string, _ int, _ []string) (string, error) { + m.EnsurePageCalls++ + if m.PageErr != nil { + return "", m.PageErr + } + return m.PagePath, nil +} + +func (m *Media) ChaptersOf(_ []string) []ports.Chapter { return m.Chapters } + +func (m *Media) PageIndex(_ context.Context, _ int64, _, _ int64, _, _ string) ([]string, error) { + if m.IndexErr != nil { + return nil, m.IndexErr + } + return m.Pages, nil +} + +func (m *Media) CacheBuster(size, modTS int64) string { return fmt.Sprintf("%d-%d", size, modTS) } + +// ---------- UploadSessions ---------- + +// Uploads is a programmable stand-in for the chunked-upload subsystem. +// Defaults simulate a happy session (uid "fakeuid…", LibraryID 1); +// hooks force error branches. +type Uploads struct { + UID string // returned by Init + LibID int64 // returned by LibraryID + Received []int64 // returned by Status + RelPath string // returned by Complete + InitErr error + LibErr error + StatusErr error + PutErr error + CompleteErr error + + PutCalls []int64 // part indices passed to PutPart + SweepCalls int + UniqueCalls int +} + +func NewUploads() *Uploads { + return &Uploads{UID: "0123456789abcdef0123456789abcdef", LibID: 1, RelPath: "book.cbz"} +} + +func (u *Uploads) Init(_ context.Context, _ int64, _ string, _, _ int64) (string, error) { + if u.InitErr != nil { + return "", u.InitErr + } + return u.UID, nil +} + +func (u *Uploads) LibraryID(_ context.Context, _ string) (int64, error) { + if u.LibErr != nil { + return 0, u.LibErr + } + return u.LibID, nil +} + +func (u *Uploads) Status(_ context.Context, _ string) ([]int64, error) { + if u.StatusErr != nil { + return nil, u.StatusErr + } + return u.Received, nil +} + +func (u *Uploads) PutPart(_ context.Context, _ string, index int64, body io.Reader, _ int64) error { + u.PutCalls = append(u.PutCalls, index) + if body != nil { // drain so callers using pipes don't block + io.Copy(io.Discard, body) + } + return u.PutErr +} + +func (u *Uploads) Complete(_ context.Context, _, _ string) (string, error) { + if u.CompleteErr != nil { + return "", u.CompleteErr + } + return u.RelPath, nil +} + +func (u *Uploads) Sweep(_ context.Context) error { + u.SweepCalls++ + return nil +} + +func (u *Uploads) UniquePath(root, name string) (string, error) { + u.UniqueCalls++ + return root + "/" + name, nil +} diff --git a/backend/internal/ports/portsfake/store.go b/backend/internal/ports/portsfake/store.go new file mode 100644 index 0000000..4c270e3 --- /dev/null +++ b/backend/internal/ports/portsfake/store.go @@ -0,0 +1,384 @@ +// Package portsfake provides hand-written, in-memory implementations of every +// interface in internal/ports. They let handler tests run without PG or Redis, +// while reproducing the real store's error semantics exactly (pgx.ErrNoRows for +// missing rows, store.ErrLastAdmin / store.ErrUniqueViolation for the guarded +// paths), so the branches under test behave as they do against the database. +// +// Fakes are safe for single-goroutine test use only; they are not locked. +package portsfake + +import ( + "context" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "booklib/internal/ports" + "booklib/internal/store" +) + +// uniqueViolation 复刻真 PG 的唯一约束冲突:store.IsUniqueViolation 只认 +// SQLSTATE 23505 的 *pgconn.PgError(生产链路里 pgx 原样透出),fake 必须同形。 +func uniqueViolation(constraint string) error { + return &pgconn.PgError{Code: "23505", ConstraintName: constraint} +} + +// compile-time proof that the fakes satisfy the consumer-side interfaces. +var ( + _ ports.UserStore = (*Users)(nil) + _ ports.LibraryStore = (*Libraries)(nil) + _ ports.BookStore = (*Books)(nil) + _ ports.ProgressStore = (*Progress)(nil) + _ ports.BookmarkStore = (*Bookmarks)(nil) + _ ports.RateLimiter = (*RateLimiter)(nil) + _ ports.Scanner = (*Scanner)(nil) + _ ports.Media = (*Media)(nil) + _ ports.UploadSessions = (*Uploads)(nil) +) + +// now is frozen per-fake-set so CreatedAt comparisons are stable within a test. +func now() time.Time { return time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) } + +// ---------- UserStore ---------- + +type Users struct { + m map[int64]store.User + next int64 +} + +func NewUsers() *Users { return &Users{m: map[int64]store.User{}, next: 1} } + +// Seed adds a user and returns its id — test convenience, not part of the port. +func (u *Users) Seed(username, hash, role string) int64 { + id := u.next + u.next++ + u.m[id] = store.User{ID: id, Username: username, PasswordHash: hash, Role: role, CreatedAt: now()} + return id +} + +func (u *Users) CountUsers(_ context.Context) (int, error) { return len(u.m), nil } + +func (u *Users) CreateUser(_ context.Context, username, hash, role string) (int64, error) { + for _, v := range u.m { + if v.Username == username { + return 0, uniqueViolation("users_username_key") + } + } + return u.Seed(username, hash, role), nil +} + +func (u *Users) GetUserByName(_ context.Context, username string) (store.User, error) { + for _, v := range u.m { + if v.Username == username { + return v, nil + } + } + return store.User{}, pgx.ErrNoRows +} + +func (u *Users) GetUserByID(_ context.Context, id int64) (store.User, error) { + if v, ok := u.m[id]; ok { + return v, nil + } + return store.User{}, pgx.ErrNoRows +} + +func (u *Users) ListUsers(_ context.Context) ([]store.User, error) { + out := make([]store.User, 0, len(u.m)) + for _, v := range u.m { + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (u *Users) DeleteUser(_ context.Context, id int64) error { + v, ok := u.m[id] + if !ok { + return pgx.ErrNoRows + } + if v.Role == "admin" { + n := 0 + for _, x := range u.m { + if x.Role == "admin" { + n++ + } + } + if n <= 1 { + return store.ErrLastAdmin + } + } + delete(u.m, id) + return nil +} + +// ---------- LibraryStore ---------- + +type Libraries struct { + m map[int64]store.Library + next int64 +} + +func NewLibraries() *Libraries { return &Libraries{m: map[int64]store.Library{}, next: 1} } + +// Seed adds a library and returns its id — test convenience. +func (l *Libraries) Seed(name, root string) int64 { + id := l.next + l.next++ + l.m[id] = store.Library{ID: id, Name: name, RootPath: root, CreatedAt: now()} + return id +} + +func (l *Libraries) CreateLibrary(_ context.Context, name, root string) (int64, error) { + for _, v := range l.m { + if v.Name == name { + return 0, uniqueViolation("libraries_name_key") + } + } + return l.Seed(name, root), nil +} + +func (l *Libraries) ListLibraries(_ context.Context) ([]store.Library, error) { + out := make([]store.Library, 0, len(l.m)) + for _, v := range l.m { + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (l *Libraries) GetLibrary(_ context.Context, id int64) (store.Library, error) { + if v, ok := l.m[id]; ok { + return v, nil + } + return store.Library{}, pgx.ErrNoRows +} + +// ---------- BookStore ---------- + +type Books struct { + m map[int64]store.Book + next int64 +} + +func NewBooks() *Books { return &Books{m: map[int64]store.Book{}, next: 1} } + +// Seed adds a book and returns its id — test convenience. +func (b *Books) Seed(libID int64, path, title, format string, size, modTS int64, pageCount int) int64 { + id := b.next + b.next++ + b.m[id] = store.Book{ID: id, LibraryID: libID, Path: path, Title: title, Format: format, + FileSize: size, ModTS: modTS, PageCount: pageCount, State: "ok", AddedAt: now()} + return id +} + +func (b *Books) InsertBook(_ context.Context, libID int64, path, title, format string, size, modTS int64, pageCount int) (int64, error) { + return b.Seed(libID, path, title, format, size, modTS, pageCount), nil +} + +func (b *Books) GetBook(_ context.Context, id int64) (store.Book, error) { + if v, ok := b.m[id]; ok { + return v, nil + } + return store.Book{}, pgx.ErrNoRows +} + +func (b *Books) ListBookMeta(_ context.Context, libID int64) (map[string]store.BookMeta, error) { + out := map[string]store.BookMeta{} + for _, v := range b.m { + if v.LibraryID == libID { + out[v.Path] = store.BookMeta{ID: v.ID, Size: v.FileSize, ModTS: v.ModTS, Format: v.Format} + } + } + return out, nil +} + +func (b *Books) UpdateBookFile(_ context.Context, id, size, modTS int64, pageCount int) error { + v, ok := b.m[id] + if !ok { + return pgx.ErrNoRows + } + v.FileSize, v.ModTS, v.PageCount = size, modTS, pageCount + b.m[id] = v + return nil +} + +func (b *Books) DeleteBookByPath(_ context.Context, libID int64, path string) error { + for id, v := range b.m { + if v.LibraryID == libID && v.Path == path { + delete(b.m, id) + return nil + } + } + return nil +} + +func (b *Books) DeleteBook(_ context.Context, id int64) error { + if _, ok := b.m[id]; !ok { + return pgx.ErrNoRows + } + delete(b.m, id) + return nil +} + +func (b *Books) SetBookState(_ context.Context, id int64, state, msg string) error { + v, ok := b.m[id] + if !ok { + return pgx.ErrNoRows + } + v.State, v.ErrMsg = state, msg + b.m[id] = v + return nil +} + +// ListBooks applies the same q/prefix filter as the SQL view (q matches title or +// path, prefix matches path prefix), then joins progress + library name. +func (b *Books) ListBooks(_ context.Context, libID int64, q, prefix string, userID int64) ([]store.BookView, error) { + out := []store.BookView{} + q = strings.ToLower(q) + for _, v := range b.m { + if libID != 0 && v.LibraryID != libID { + continue + } + if prefix != "" && !strings.HasPrefix(v.Path, prefix) { + continue + } + if q != "" && !strings.Contains(strings.ToLower(v.Title), q) && !strings.Contains(strings.ToLower(v.Path), q) { + continue + } + out = append(out, store.BookView{Book: v}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Book.ID < out[j].Book.ID }) + return out, nil +} + +func (b *Books) BookHashes(_ context.Context) (map[int64][2]int64, error) { + out := map[int64][2]int64{} + for _, v := range b.m { + out[v.ID] = [2]int64{v.FileSize, v.ModTS} + } + return out, nil +} + +// ---------- ProgressStore ---------- + +type progressKey struct { + userID, libID int64 + bookPath string +} + +type Progress struct { + m map[progressKey]store.Progress + libs *Libraries // for LibraryName join; may be nil + books *Books // for Title join; may be nil +} + +func NewProgress(libs *Libraries, books *Books) *Progress { + return &Progress{m: map[progressKey]store.Progress{}, libs: libs, books: books} +} + +func (p *Progress) UpsertProgress(_ context.Context, userID, libID int64, bookPath string, locator []byte, percent float64) error { + k := progressKey{userID, libID, bookPath} + old := p.m[k] + p.m[k] = store.Progress{LibraryID: libID, BookPath: bookPath, Locator: locator, + Percent: percent, UpdatedAt: now(), LibraryName: old.LibraryName, Title: old.Title} + return nil +} + +func (p *Progress) GetProgress(_ context.Context, userID, libID int64, bookPath string) (store.Progress, error) { + if v, ok := p.m[progressKey{userID, libID, bookPath}]; ok { + return v, nil + } + return store.Progress{}, pgx.ErrNoRows +} + +func (p *Progress) ListProgress(_ context.Context, userID int64) ([]store.Progress, error) { + out := []store.Progress{} + for k, v := range p.m { + if k.userID != userID { + continue + } + if p.libs != nil { + if l, e := p.libs.GetLibrary(context.Background(), v.LibraryID); e == nil { + v.LibraryName = l.Name + } + } + if p.books != nil { + for _, b := range p.books.m { + if b.LibraryID == v.LibraryID && b.Path == v.BookPath { + v.Title = b.Title + break + } + } + } + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].BookPath < out[j].BookPath }) + return out, nil +} + +// ---------- BookmarkStore ---------- + +// bookmark pairs a row with its owner id. The real store carries the owner in a +// users-scoped join; the fake keeps it alongside so ListBookmarks/Patch/Delete +// can honour owner-scoped 404 semantics without a global index. +type bookmark struct { + row store.Bookmark + userID int64 +} + +type Bookmarks struct { + m map[int64]bookmark + next int64 +} + +func NewBookmarks() *Bookmarks { return &Bookmarks{m: map[int64]bookmark{}, next: 1} } + +func (bm *Bookmarks) InsertBookmark(_ context.Context, userID, libID int64, bookPath string, locator []byte, percent float64, note string) (int64, error) { + id := bm.next + bm.next++ + bm.m[id] = bookmark{ + row: store.Bookmark{ID: id, LibraryID: libID, BookPath: bookPath, + Locator: locator, Percent: percent, Note: note, CreatedAt: now()}, + userID: userID, + } + return id, nil +} + +func (bm *Bookmarks) ListBookmarks(_ context.Context, userID, libID int64, bookPath string) ([]store.Bookmark, error) { + out := []store.Bookmark{} + for _, v := range bm.m { + if v.userID == userID && v.row.LibraryID == libID && v.row.BookPath == bookPath { + out = append(out, v.row) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Percent != out[j].Percent { + return out[i].Percent < out[j].Percent + } + return out[i].ID < out[j].ID + }) + return out, nil +} + +func (bm *Bookmarks) UpdateBookmarkNote(_ context.Context, userID, id int64, note string) (bool, error) { + v, ok := bm.m[id] + if !ok || v.userID != userID { // owner-scoped: foreign id is a 404, not a 403 + return false, nil + } + v.row.Note = note + bm.m[id] = v + return true, nil +} + +func (bm *Bookmarks) DeleteBookmark(_ context.Context, userID, id int64) (bool, error) { + v, ok := bm.m[id] + if !ok || v.userID != userID { + return false, nil + } + delete(bm.m, id) + return true, nil +} diff --git a/backend/internal/redispkg/redis.go b/backend/internal/redispkg/redis.go index abae72e..f698c8d 100644 --- a/backend/internal/redispkg/redis.go +++ b/backend/internal/redispkg/redis.go @@ -44,18 +44,25 @@ func (r *R) Set(ctx context.Context, key, val string, ttl time.Duration) { } } +// incrWindowScript atomically increments and sets TTL on first value, +// preventing the INCR+EXPIRE race that could leave keys without TTL (B1). +var incrWindowScript = redis.NewScript(` +local n = redis.call('INCR', KEYS[1]) +if n == 1 then + redis.call('EXPIRE', KEYS[1], ARGV[1]) +end +return n +`) + func (r *R) IncrWindow(ctx context.Context, key string, ttl time.Duration) int { if r.c == nil { return 1 } - n, err := r.c.Incr(ctx, key).Result() + n, err := incrWindowScript.Run(ctx, r.c, []string{key}, int(ttl.Seconds())).Int() if err != nil { - return 1 + return 1 // fail-open } - if n == 1 { - r.c.Expire(ctx, key, ttl) - } - return int(n) + return n } func (r *R) Lock(ctx context.Context, key string, ttl time.Duration) (func(), bool) { @@ -64,7 +71,11 @@ func (r *R) Lock(ctx context.Context, key string, ttl time.Duration) (func(), bo return noop, true } b := make([]byte, 8) - rand.Read(b) + if _, err := rand.Read(b); err != nil { + // B2: rand failure → degrade to no-lock instead of using a zero token. + log.Printf("rand.Read failed: %v (proceeding without lock)", err) + return noop, true + } tok := hex.EncodeToString(b) ok, err := r.c.SetNX(ctx, key, tok, ttl).Result() if err != nil { // spec §9: Redis 故障降级放行,锁只做尽力去重 @@ -75,8 +86,68 @@ func (r *R) Lock(ctx context.Context, key string, ttl time.Duration) (func(), bo return noop, false // 锁被持有,别的副本在扫 } return func() { - r.c.Eval(ctx, + // B3: use WithoutCancel so unlock survives caller cancellation. + if err := r.c.Eval(context.WithoutCancel(ctx), "if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", - []string{key}, tok) + []string{key}, tok).Err(); err != nil { + log.Printf("redis unlock %s: %v", key, err) + } }, true } + +// ScanLock acquires a distributed lock with automatic renewal. +// The lock is renewed every ttl/2 until unlock is called. +// Returns (unlock, true) on success, (noop, true) on redis failure (degrade), +// or (noop, false) if the lock is already held. +func (r *R) ScanLock(ctx context.Context, key string, ttl time.Duration) (func(), bool) { + noop := func() {} + if r.c == nil { + return noop, true + } + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + log.Printf("rand.Read failed: %v (proceeding without lock)", err) + return noop, true + } + tok := hex.EncodeToString(b) + ok, err := r.c.SetNX(ctx, key, tok, ttl).Result() + if err != nil { + log.Printf("redis scanlock %s: %v (proceeding without lock)", key, err) + return noop, true + } + if !ok { + return noop, false + } + + // Start renewal goroutine. + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(ttl / 2) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + // Renew only if we still own the lock. + if err := r.c.Eval(context.Background(), + `if redis.call('get',KEYS[1])==ARGV[1] then + return redis.call('expire',KEYS[1],ARGV[2]) + else return 0 end`, + []string{key}, tok, int(ttl.Seconds())).Err(); err != nil { + log.Printf("redis scanlock renew %s: %v", key, err) + } + } + } + }() + + unlock := func() { + close(done) // stop renewal + if err := r.c.Eval(context.WithoutCancel(ctx), + "if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", + []string{key}, tok).Err(); err != nil { + log.Printf("redis scanlock unlock %s: %v", key, err) + } + } + return unlock, true +} diff --git a/backend/internal/redispkg/redis_test.go b/backend/internal/redispkg/redis_test.go index 8d65092..62dbf1f 100644 --- a/backend/internal/redispkg/redis_test.go +++ b/backend/internal/redispkg/redis_test.go @@ -2,6 +2,7 @@ package redispkg import ( "context" + "os" "testing" "time" ) @@ -21,6 +22,12 @@ func TestDisabledIsSafe(t *testing.T) { t.Fatal("disabled Lock must always acquire") } un() + // ScanLock disabled mode. + un2, ok2 := r.ScanLock(ctx, "slk", time.Second) + if !ok2 { + t.Fatal("disabled ScanLock must always acquire") + } + un2() } func TestDeadRedisLockFailsOpen(t *testing.T) { @@ -31,3 +38,96 @@ func TestDeadRedisLockFailsOpen(t *testing.T) { } un() } + +func TestDeadRedisScanLockFailsOpen(t *testing.T) { + r := New("redis://127.0.0.1:16399") + un, ok := r.ScanLock(context.Background(), "slk", time.Second) + if !ok { + t.Fatal("ScanLock on redis error must fail open") + } + un() +} + +func newLiveRedis(t *testing.T) *R { + t.Helper() + url := os.Getenv("REDIS_URL") + if url == "" { + t.Skip("REDIS_URL not set") + } + return New(url) +} + +func TestIncrWindowSetsTTL(t *testing.T) { + r := newLiveRedis(t) + ctx := context.Background() + key := "test:incrwindow:ttl:" + t.Name() + r.c.Del(ctx, key) + + n := r.IncrWindow(ctx, key, 5*time.Second) + if n != 1 { + t.Fatalf("first call = %d, want 1", n) + } + ttl, err := r.c.TTL(ctx, key).Result() + if err != nil { + t.Fatal(err) + } + if ttl <= 0 { + t.Fatalf("TTL should be positive after first increment, got %v", ttl) + } + + // Second call: n=2, TTL should still be positive. + n = r.IncrWindow(ctx, key, 5*time.Second) + if n != 2 { + t.Fatalf("second call = %d, want 2", n) + } + r.c.Del(ctx, key) +} + +func TestScanLockRenewal(t *testing.T) { + r := newLiveRedis(t) + ctx := context.Background() + key := "test:scanlock:" + t.Name() + r.c.Del(ctx, key) + + unlock, ok := r.ScanLock(ctx, key, 2*time.Second) + if !ok { + t.Fatal("should acquire") + } + // Wait 3 seconds — without renewal, the lock would expire at 2s. + time.Sleep(3 * time.Second) + + // A second attempt should fail (lock still held by first, renewed). + _, ok2 := r.ScanLock(ctx, key, 2*time.Second) + if ok2 { + t.Fatal("second acquire should fail — lock should have been renewed") + } + unlock() + r.c.Del(ctx, key) +} + +func TestScanLockMutualExclusion(t *testing.T) { + r := newLiveRedis(t) + ctx := context.Background() + key := "test:scanlock:mutex:" + t.Name() + r.c.Del(ctx, key) + + unlock1, ok1 := r.ScanLock(ctx, key, 10*time.Second) + if !ok1 { + t.Fatal("first should acquire") + } + + _, ok2 := r.ScanLock(ctx, key, 10*time.Second) + if ok2 { + t.Fatal("second should not acquire while first holds") + } + + unlock1() + // After unlock, a new acquire should succeed. + time.Sleep(50 * time.Millisecond) + unlock3, ok3 := r.ScanLock(ctx, key, 10*time.Second) + if !ok3 { + t.Fatal("should acquire after unlock") + } + unlock3() + r.c.Del(ctx, key) +} diff --git a/backend/internal/scanner/scanner.go b/backend/internal/scanner/scanner.go index e794c52..41bea9b 100644 --- a/backend/internal/scanner/scanner.go +++ b/backend/internal/scanner/scanner.go @@ -7,25 +7,35 @@ import ( "io" "io/fs" "log" - "os" "path/filepath" "strings" + "sync" "time" "booklib/internal/bookfile" "booklib/internal/config" + "booklib/internal/media" "booklib/internal/redispkg" "booklib/internal/store" ) -type Scanner struct { - st *store.Store - cfg *config.Config - rdb *redispkg.R +// Sweeper 是 scanner 每轮顺手调用的清理钩子;upload.U 满足它(B16)。 +type Sweeper interface { + Sweep(ctx context.Context) error } -func New(st *store.Store, cfg *config.Config, rdb *redispkg.R) *Scanner { - return &Scanner{st: st, cfg: cfg, rdb: rdb} +type Scanner struct { + st *store.Store + cfg *config.Config + rdb *redispkg.R + sweepers []Sweeper + // B9-②: per-library single-flight — concurrent scan triggers for the same + // library are merged into one execution, even without redis. + flights sync.Map // map[int64]*sync.WaitGroup +} + +func New(st *store.Store, cfg *config.Config, rdb *redispkg.R, sweepers ...Sweeper) *Scanner { + return &Scanner{st: st, cfg: cfg, rdb: rdb, sweepers: sweepers} } func (s *Scanner) Run(ctx context.Context) { @@ -36,13 +46,18 @@ func (s *Scanner) Run(ctx context.Context) { case <-ctx.Done(): return case <-t.C: + for _, sw := range s.sweepers { // B16: 上传会话清扫随扫描周期跑 + if err := sw.Sweep(ctx); err != nil { + log.Printf("scan: sweep: %v", err) + } + } libs, err := s.st.ListLibraries(ctx) if err != nil { log.Printf("scan: list libraries: %v", err) continue } for _, l := range libs { - s.ScanLibrary(ctx, l) + s.scanOnce(ctx, l) } } } @@ -54,19 +69,35 @@ func (s *Scanner) ScanLibraryByID(ctx context.Context, id int64) { log.Printf("scan: library %d: %v", id, err) return } + s.scanOnce(ctx, lib) +} + +// scanOnce ensures only one scan per library runs concurrently in this process. +// Concurrent callers block until the in-flight scan completes (B9-②). +func (s *Scanner) scanOnce(ctx context.Context, lib store.Library) { + wg := &sync.WaitGroup{} + wg.Add(1) + if existing, loaded := s.flights.LoadOrStore(lib.ID, wg); loaded { + existing.(*sync.WaitGroup).Wait() + return + } + defer func() { + s.flights.Delete(lib.ID) + wg.Done() + }() s.ScanLibrary(ctx, lib) } func (s *Scanner) ScanLibrary(ctx context.Context, lib store.Library) { - // ponytail: 5min lock TTL; a scan longer than this lets another replica join — refresh mid-walk if libs ever outgrow it - unlock, ok := s.rdb.Lock(ctx, fmt.Sprintf("scan:%d", lib.ID), 5*time.Minute) + // B9-①: ScanLock auto-renews every TTL/2 during long scans. + unlock, ok := s.rdb.ScanLock(ctx, fmt.Sprintf("scan:%d", lib.ID), 5*time.Minute) if !ok { return // 别的副本在扫 } defer unlock() root, err := filepath.EvalSymlinks(filepath.Clean(lib.RootPath)) - if err != nil || !inside(s.cfg.BooksDir, root) { + if err != nil || !bookfile.Contains(s.cfg.BooksDir, root) { log.Printf("scan: library %d root %q rejected", lib.ID, lib.RootPath) return } @@ -85,9 +116,9 @@ func (s *Scanner) ScanLibrary(ctx context.Context, lib store.Library) { delete(dbMeta, rel) switch { case !exists: - s.add(ctx, lib.ID, root, rel, ds) + s.ingest(ctx, lib.ID, 0, root, rel, ds, true) case old.Size != ds.size || old.ModTS != ds.modTS: - s.update(ctx, lib.ID, old.ID, root, rel, ds) + s.ingest(ctx, lib.ID, old.ID, root, rel, ds, false) } } for rel := range dbMeta { // 只剩被删的文件 @@ -100,11 +131,6 @@ func (s *Scanner) ScanLibrary(ctx context.Context, lib store.Library) { type diskStat struct{ size, modTS int64 } -func inside(booksDir, root string) bool { - b := filepath.Clean(booksDir) - return root == b || strings.HasPrefix(root, b+string(os.PathSeparator)) -} - func walk(root string) (map[string]diskStat, error) { out := map[string]diskStat{} err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { @@ -137,9 +163,9 @@ func titleOf(rel string) string { return strings.TrimSpace(strings.ReplaceAll(strings.TrimSuffix(base, filepath.Ext(base)), "_", " ")) } -// cbz 完整性判定集中在 add/update:PageIndex 失败 → state=error。 -// InsertBook/UpdateBookFile 的 SQL 已把 state 重置为 ready(Task 2),无需显式清 error。 -func (s *Scanner) add(ctx context.Context, libID int64, root, rel string, ds diskStat) { +// ingest 是 add/update 的合一实现(Task 24):isNew 决定走 Insert 还是 UpdateBookFile, +// 之后的错误处理与封面生成完全共享。bookID 仅在 isNew=false 时有意义。 +func (s *Scanner) ingest(ctx context.Context, libID, bookID int64, root, rel string, ds diskStat, isNew bool) { format := bookfile.FormatFromExt(filepath.Base(rel)) pageCount := 0 var idxErr error @@ -147,98 +173,67 @@ func (s *Scanner) add(ctx context.Context, libID int64, root, rel string, ds dis idx, err := s.zipIndex(root, rel) pageCount = len(idx) idxErr = err - if idxErr == nil && pageCount == 0 { // 视频/文档 zip 不是漫画,空白 reader 没有意义 + if idxErr == nil && pageCount == 0 { idxErr = errors.New("no images in archive") } } - id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount) - if err != nil { - log.Printf("scan: insert %s: %v", rel, err) - return - } - if idxErr != nil { - s.st.SetBookState(ctx, id, "error", idxErr.Error()) - return - } - s.cover(ctx, id, root, rel, format, ds) -} - -func (s *Scanner) update(ctx context.Context, libID, bookID int64, root, rel string, ds diskStat) { - format := bookfile.FormatFromExt(filepath.Base(rel)) - pageCount := 0 - var idxErr error - if format == "cbz" { - idx, err := s.zipIndex(root, rel) - pageCount = len(idx) - idxErr = err - if idxErr == nil && pageCount == 0 { // 视频/文档 zip 不是漫画,空白 reader 没有意义 - idxErr = errors.New("no images in archive") + if isNew { + id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount) + if err != nil { + log.Printf("scan: insert %s: %v", rel, err) + return } - } - if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil { + bookID = id + } else if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil { log.Printf("scan: update %s: %v", rel, err) return } if idxErr != nil { - s.st.SetBookState(ctx, bookID, "error", idxErr.Error()) + // B10: log SetBookState errors instead of discarding. + if e := s.st.SetBookState(ctx, bookID, "error", idxErr.Error()); e != nil { + log.Printf("scan: SetBookState %s: %v", rel, e) + } return } s.cover(ctx, bookID, root, rel, format, ds) } func (s *Scanner) zipIndex(root, rel string) ([]string, error) { - f, err := os.Open(filepath.Join(root, filepath.FromSlash(rel))) + f, size, err := bookfile.OpenReaderAt(root, rel) if err != nil { return nil, err } defer f.Close() - st, err := f.Stat() - if err != nil { - return nil, err - } - return bookfile.PageIndex(f, st.Size()) + return bookfile.PageIndex(f, size) } -// cover 失败(坏 epub、无图等)只 log — 书的 state 由 PageIndex 判定,封面缺了有占位 SVG 兜底 +// cover writes the cover image to the cache dir via media.WriteAtomic. +// B11: write failures are logged; orphan .tmp files are cleaned only on failure. func (s *Scanner) cover(ctx context.Context, id int64, root, rel, format string, ds diskStat) { - var img []byte - var ext string - var err error + var fn func(io.ReaderAt, int64) ([]byte, string, error) switch format { case "cbz": - img, ext, err = s.readCover(root, rel, bookfile.CBZCover) + fn = bookfile.CBZCover case "epub": - img, ext, err = s.readCover(root, rel, bookfile.EPUBCover) + fn = bookfile.EPUBCover default: return // pdf/txt/md 用占位 SVG,不落盘 } + f, size, err := bookfile.OpenReaderAt(root, rel) + if err != nil { + log.Printf("scan: cover %s: %v", rel, err) + return + } + defer f.Close() + img, ext, err := fn(f, size) if err != nil { log.Printf("scan: cover %s: %v", rel, err) return } dir := bookfile.CoverDir(s.cfg.CacheDir, bookfile.DirKey(id, bookfile.Hash(ds.size, ds.modTS))) - if e := os.MkdirAll(dir, 0o755); e != nil { - log.Printf("scan: coverdir %s: %v", rel, e) - return + if e := media.WriteAtomic(dir, "cover"+ext, img); e != nil { + log.Printf("scan: cover write %s: %v", rel, e) } - tmp := filepath.Join(dir, "cover"+ext+".tmp") - dst := filepath.Join(dir, "cover"+ext) - if e := os.WriteFile(tmp, img, 0o644); e == nil { - os.Rename(tmp, dst) - } -} - -func (s *Scanner) readCover(root, rel string, fn func(io.ReaderAt, int64) ([]byte, string, error)) ([]byte, string, error) { - f, err := os.Open(filepath.Join(root, filepath.FromSlash(rel))) - if err != nil { - return nil, "", err - } - defer f.Close() - st, err := f.Stat() - if err != nil { - return nil, "", err - } - return fn(f, st.Size()) } func (s *Scanner) sweepCache(ctx context.Context) { diff --git a/backend/internal/seed/seed.go b/backend/internal/seed/seed.go index 168f2b0..1cd4418 100644 --- a/backend/internal/seed/seed.go +++ b/backend/internal/seed/seed.go @@ -2,11 +2,8 @@ package seed import ( "context" - "errors" "log" - "github.com/jackc/pgx/v5/pgconn" - "booklib/internal/auth" "booklib/internal/store" ) @@ -28,8 +25,7 @@ func Admin(ctx context.Context, s *store.Store, user, pass string) error { return err } if _, err := s.CreateUser(ctx, user, h, "admin"); err != nil { - var pgErr *pgconn.PgError - if errors.As(err, &pgErr) && pgErr.Code == "23505" { + if store.IsUniqueViolation(err) { log.Printf("seed admin %q may already exist: %v", user, err) return nil } diff --git a/backend/internal/store/bookmarks.go b/backend/internal/store/bookmarks.go new file mode 100644 index 0000000..88749ef --- /dev/null +++ b/backend/internal/store/bookmarks.go @@ -0,0 +1,48 @@ +package store + +import "context" + +func (s *Store) InsertBookmark(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64, note string) (int64, error) { + var id int64 + err := s.p.QueryRow(ctx, + `INSERT INTO bookmarks (user_id, library_id, book_path, locator, percent, note) + VALUES ($1,$2,$3,$4,$5,$6) RETURNING id`, + userID, libID, bookPath, locator, percent, note).Scan(&id) + return id, err +} + +func (s *Store) ListBookmarks(ctx context.Context, userID, libID int64, bookPath string) ([]Bookmark, error) { + rows, err := s.p.Query(ctx, + `SELECT id, library_id, book_path, locator, percent, note, created_at + FROM bookmarks WHERE user_id=$1 AND library_id=$2 AND book_path=$3 + ORDER BY percent ASC, id ASC`, userID, libID, bookPath) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Bookmark + for rows.Next() { + var b Bookmark + if err := rows.Scan(&b.ID, &b.LibraryID, &b.BookPath, &b.Locator, &b.Percent, &b.Note, &b.CreatedAt); err != nil { + return nil, err + } + out = append(out, b) + } + return out, rows.Err() +} + +func (s *Store) UpdateBookmarkNote(ctx context.Context, userID, id int64, note string) (bool, error) { + res, err := s.p.Exec(ctx, `UPDATE bookmarks SET note=$3 WHERE id=$1 AND user_id=$2`, id, userID, note) + if err != nil { + return false, err + } + return res.RowsAffected() > 0, nil +} + +func (s *Store) DeleteBookmark(ctx context.Context, userID, id int64) (bool, error) { + res, err := s.p.Exec(ctx, `DELETE FROM bookmarks WHERE id=$1 AND user_id=$2`, id, userID) + if err != nil { + return false, err + } + return res.RowsAffected() > 0, nil +} diff --git a/backend/internal/store/books.go b/backend/internal/store/books.go new file mode 100644 index 0000000..c9f6128 --- /dev/null +++ b/backend/internal/store/books.go @@ -0,0 +1,109 @@ +package store + +import "context" + +const bookCols = "id, library_id, path, title, format, file_size, mod_ts, page_count, state, error_msg, added_at" + +func (s *Store) InsertBook(ctx context.Context, libID int64, path, title, format string, size, modTS int64, pageCount int) (int64, error) { + var id int64 + err := s.p.QueryRow(ctx, + `INSERT INTO books (library_id, path, title, format, file_size, mod_ts, page_count) + VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING id`, + libID, path, title, format, size, modTS, pageCount).Scan(&id) + return id, err +} + +func (s *Store) GetBook(ctx context.Context, id int64) (Book, error) { + var b Book + err := s.p.QueryRow(ctx, "SELECT "+bookCols+" FROM books WHERE id=$1", id).Scan( + &b.ID, &b.LibraryID, &b.Path, &b.Title, &b.Format, + &b.FileSize, &b.ModTS, &b.PageCount, &b.State, &b.ErrMsg, &b.AddedAt) + return b, err +} + +func (s *Store) ListBookMeta(ctx context.Context, libID int64) (map[string]BookMeta, error) { + rows, err := s.p.Query(ctx, + "SELECT id, path, file_size, mod_ts, format FROM books WHERE library_id=$1", libID) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]BookMeta{} + for rows.Next() { + var m BookMeta + var path string + if err := rows.Scan(&m.ID, &path, &m.Size, &m.ModTS, &m.Format); err != nil { + return nil, err + } + out[path] = m + } + return out, rows.Err() +} + +func (s *Store) UpdateBookFile(ctx context.Context, id, size, modTS int64, pageCount int) error { + _, err := s.p.Exec(ctx, + `UPDATE books SET file_size=$2, mod_ts=$3, page_count=$4, state='ready', error_msg='' WHERE id=$1`, + id, size, modTS, pageCount) + return err +} + +func (s *Store) DeleteBookByPath(ctx context.Context, libID int64, path string) error { + _, err := s.p.Exec(ctx, "DELETE FROM books WHERE library_id=$1 AND path=$2", libID, path) + return err +} + +func (s *Store) DeleteBook(ctx context.Context, id int64) error { + _, err := s.p.Exec(ctx, "DELETE FROM books WHERE id=$1", id) + return err +} + +func (s *Store) SetBookState(ctx context.Context, id int64, state, msg string) error { + _, err := s.p.Exec(ctx, "UPDATE books SET state=$2, error_msg=$3 WHERE id=$1", id, state, msg) + return err +} + +func (s *Store) ListBooks(ctx context.Context, libID int64, q, prefix string, userID int64) ([]BookView, error) { + rows, err := s.p.Query(ctx, + `SELECT b.id, b.library_id, b.path, b.title, b.format, b.file_size, b.mod_ts, + b.page_count, b.state, b.error_msg, b.added_at, l.name, COALESCE(p.percent, 0) + FROM books b JOIN libraries l ON l.id = b.library_id + LEFT JOIN reading_progress p ON p.user_id = $4 AND p.library_id = b.library_id AND p.book_path = b.path + WHERE ($1 = 0 OR b.library_id = $1) + AND ($2 = '' OR lower(b.title) LIKE '%' || lower($2) || '%') + AND ($3 = '' OR b.path LIKE $3 || '%') + ORDER BY l.name, b.path`, libID, q, prefix, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []BookView + for rows.Next() { + var v BookView + err := rows.Scan(&v.ID, &v.LibraryID, &v.Path, &v.Title, &v.Format, + &v.FileSize, &v.ModTS, &v.PageCount, &v.State, &v.ErrMsg, &v.AddedAt, + &v.LibraryName, &v.Percent) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +func (s *Store) BookHashes(ctx context.Context) (map[int64][2]int64, error) { + rows, err := s.p.Query(ctx, "SELECT id, file_size, mod_ts FROM books") + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int64][2]int64{} + for rows.Next() { + var id int64 + var v [2]int64 + if err := rows.Scan(&id, &v[0], &v[1]); err != nil { + return nil, err + } + out[id] = v + } + return out, rows.Err() +} diff --git a/backend/internal/store/libraries.go b/backend/internal/store/libraries.go new file mode 100644 index 0000000..72998cf --- /dev/null +++ b/backend/internal/store/libraries.go @@ -0,0 +1,35 @@ +package store + +import "context" + +func (s *Store) CreateLibrary(ctx context.Context, name, root string) (int64, error) { + var id int64 + err := s.p.QueryRow(ctx, + "INSERT INTO libraries (name, root_path) VALUES ($1,$2) RETURNING id", name, root).Scan(&id) + return id, err +} + +func (s *Store) ListLibraries(ctx context.Context) ([]Library, error) { + rows, err := s.p.Query(ctx, "SELECT id, name, root_path, created_at FROM libraries ORDER BY id") + if err != nil { + return nil, err + } + defer rows.Close() + var out []Library + for rows.Next() { + var l Library + if err := rows.Scan(&l.ID, &l.Name, &l.RootPath, &l.CreatedAt); err != nil { + return nil, err + } + out = append(out, l) + } + return out, rows.Err() +} + +func (s *Store) GetLibrary(ctx context.Context, id int64) (Library, error) { + var l Library + err := s.p.QueryRow(ctx, + "SELECT id, name, root_path, created_at FROM libraries WHERE id=$1", id). + Scan(&l.ID, &l.Name, &l.RootPath, &l.CreatedAt) + return l, err +} diff --git a/backend/internal/store/progress.go b/backend/internal/store/progress.go new file mode 100644 index 0000000..139b22a --- /dev/null +++ b/backend/internal/store/progress.go @@ -0,0 +1,45 @@ +package store + +import "context" + +func (s *Store) UpsertProgress(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64) error { + _, err := s.p.Exec(ctx, + `INSERT INTO reading_progress (user_id, library_id, book_path, locator, percent, updated_at) + VALUES ($1,$2,$3,$4,$5,now()) + ON CONFLICT (user_id, library_id, book_path) + DO UPDATE SET locator=$4, percent=$5, updated_at=now()`, + userID, libID, bookPath, locator, percent) + return err +} + +func (s *Store) GetProgress(ctx context.Context, userID, libID int64, bookPath string) (Progress, error) { + var pr Progress + err := s.p.QueryRow(ctx, + `SELECT library_id, book_path, locator, percent, updated_at + FROM reading_progress WHERE user_id=$1 AND library_id=$2 AND book_path=$3`, + userID, libID, bookPath). + Scan(&pr.LibraryID, &pr.BookPath, &pr.Locator, &pr.Percent, &pr.UpdatedAt) + return pr, err +} + +func (s *Store) ListProgress(ctx context.Context, userID int64) ([]Progress, error) { + rows, err := s.p.Query(ctx, + `SELECT p.library_id, l.name, p.book_path, COALESCE(b.title, ''), p.locator, p.percent, p.updated_at + FROM reading_progress p JOIN libraries l ON l.id = p.library_id + LEFT JOIN books b ON b.library_id = p.library_id AND b.path = p.book_path + WHERE p.user_id = $1 ORDER BY p.updated_at DESC`, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Progress + for rows.Next() { + var pr Progress + if err := rows.Scan(&pr.LibraryID, &pr.LibraryName, &pr.BookPath, &pr.Title, + &pr.Locator, &pr.Percent, &pr.UpdatedAt); err != nil { + return nil, err + } + out = append(out, pr) + } + return out, rows.Err() +} diff --git a/backend/internal/store/store.go b/backend/internal/store/store.go index f365f97..1be2fb4 100644 --- a/backend/internal/store/store.go +++ b/backend/internal/store/store.go @@ -1,16 +1,29 @@ package store import ( - "context" + "errors" "time" - "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" ) -type Store struct{ P *pgxpool.Pool } +// ErrLastAdmin is returned by DeleteUser when attempting to delete the last admin. +var ErrLastAdmin = errors.New("cannot delete the last admin") -func New(p *pgxpool.Pool) *Store { return &Store{P: p} } +// ErrUniqueViolation is a sentinel for PostgreSQL unique constraint violations. +var ErrUniqueViolation = errors.New("unique violation") + +// IsUniqueViolation reports whether err is a PostgreSQL unique constraint violation (23505). +func IsUniqueViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23505" +} + +// Store provides database access. The pool is unexported; all access goes through methods. +type Store struct{ p *pgxpool.Pool } + +func New(p *pgxpool.Pool) *Store { return &Store{p: p} } // ---------- types ---------- @@ -21,12 +34,14 @@ type User struct { Role string CreatedAt time.Time } + type Library struct { ID int64 Name string RootPath string CreatedAt time.Time } + type Book struct { ID, LibraryID int64 Path string @@ -34,288 +49,35 @@ type Book struct { Format string FileSize int64 ModTS int64 - PageCount int // 0 表示未知(pdf/epub/txt) + PageCount int // 0 means unknown (pdf/epub/txt) State string ErrMsg string AddedAt time.Time } + type BookMeta struct { ID int64 Size int64 ModTS int64 Format string } + type BookView struct { Book LibraryName string Percent float64 } + type Progress struct { LibraryID int64 LibraryName string BookPath string - Title string // 书已删时为空 + Title string // empty when book deleted Locator []byte Percent float64 UpdatedAt time.Time } -// ---------- users ---------- - -const userCols = "id, username, password_hash, role, created_at" - -func (s *Store) CountUsers(ctx context.Context) (int, error) { - var n int - err := s.P.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&n) - return n, err -} - -func (s *Store) CreateUser(ctx context.Context, username, hash, role string) (int64, error) { - var id int64 - err := s.P.QueryRow(ctx, - "INSERT INTO users (username, password_hash, role) VALUES ($1,$2,$3) RETURNING id", - username, hash, role).Scan(&id) - return id, err -} - -func (s *Store) GetUserByName(ctx context.Context, username string) (User, error) { - return scanUser(s.P.QueryRow(ctx, "SELECT "+userCols+" FROM users WHERE username=$1", username)) -} - -func (s *Store) GetUserByID(ctx context.Context, id int64) (User, error) { - return scanUser(s.P.QueryRow(ctx, "SELECT "+userCols+" FROM users WHERE id=$1", id)) -} - -func (s *Store) ListUsers(ctx context.Context) ([]User, error) { - rows, err := s.P.Query(ctx, "SELECT "+userCols+" FROM users ORDER BY id") - if err != nil { - return nil, err - } - defer rows.Close() - var out []User - for rows.Next() { - var u User - if err := rows.Scan(&u.ID, &u.Username, &u.PasswordHash, &u.Role, &u.CreatedAt); err != nil { - return nil, err - } - out = append(out, u) - } - return out, rows.Err() -} - -func (s *Store) DeleteUser(ctx context.Context, id int64) error { - _, err := s.P.Exec(ctx, "DELETE FROM users WHERE id=$1", id) - return err -} - -func scanUser(row pgx.Row) (User, error) { - var u User - err := row.Scan(&u.ID, &u.Username, &u.PasswordHash, &u.Role, &u.CreatedAt) - return u, err -} - -// CountAdmins 供 Task 5 的"最后一个 admin 不可删"保护 -func (s *Store) CountAdmins(ctx context.Context) (int, error) { - var n int - err := s.P.QueryRow(ctx, "SELECT count(*) FROM users WHERE role='admin'").Scan(&n) - return n, err -} - -// ---------- libraries ---------- - -func (s *Store) CreateLibrary(ctx context.Context, name, root string) (int64, error) { - var id int64 - err := s.P.QueryRow(ctx, - "INSERT INTO libraries (name, root_path) VALUES ($1,$2) RETURNING id", name, root).Scan(&id) - return id, err -} - -func (s *Store) ListLibraries(ctx context.Context) ([]Library, error) { - rows, err := s.P.Query(ctx, "SELECT id, name, root_path, created_at FROM libraries ORDER BY id") - if err != nil { - return nil, err - } - defer rows.Close() - var out []Library - for rows.Next() { - var l Library - if err := rows.Scan(&l.ID, &l.Name, &l.RootPath, &l.CreatedAt); err != nil { - return nil, err - } - out = append(out, l) - } - return out, rows.Err() -} - -func (s *Store) GetLibrary(ctx context.Context, id int64) (Library, error) { - var l Library - err := s.P.QueryRow(ctx, - "SELECT id, name, root_path, created_at FROM libraries WHERE id=$1", id). - Scan(&l.ID, &l.Name, &l.RootPath, &l.CreatedAt) - return l, err -} - -// ---------- books ---------- - -const bookCols = "id, library_id, path, title, format, file_size, mod_ts, page_count, state, error_msg, added_at" - -func (s *Store) InsertBook(ctx context.Context, libID int64, path, title, format string, size, modTS int64, pageCount int) (int64, error) { - var id int64 - err := s.P.QueryRow(ctx, - `INSERT INTO books (library_id, path, title, format, file_size, mod_ts, page_count) - VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING id`, - libID, path, title, format, size, modTS, pageCount).Scan(&id) - return id, err -} - -func (s *Store) GetBook(ctx context.Context, id int64) (Book, error) { - var b Book - err := s.P.QueryRow(ctx, "SELECT "+bookCols+" FROM books WHERE id=$1", id).Scan( - &b.ID, &b.LibraryID, &b.Path, &b.Title, &b.Format, - &b.FileSize, &b.ModTS, &b.PageCount, &b.State, &b.ErrMsg, &b.AddedAt) - return b, err -} - -func (s *Store) ListBookMeta(ctx context.Context, libID int64) (map[string]BookMeta, error) { - rows, err := s.P.Query(ctx, - "SELECT id, path, file_size, mod_ts, format FROM books WHERE library_id=$1", libID) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]BookMeta{} - for rows.Next() { - var m BookMeta - var path string - if err := rows.Scan(&m.ID, &path, &m.Size, &m.ModTS, &m.Format); err != nil { - return nil, err - } - out[path] = m - } - return out, rows.Err() -} - -func (s *Store) UpdateBookFile(ctx context.Context, id, size, modTS int64, pageCount int) error { - _, err := s.P.Exec(ctx, - `UPDATE books SET file_size=$2, mod_ts=$3, page_count=$4, state='ready', error_msg='' WHERE id=$1`, - id, size, modTS, pageCount) - return err -} - -func (s *Store) DeleteBookByPath(ctx context.Context, libID int64, path string) error { - _, err := s.P.Exec(ctx, "DELETE FROM books WHERE library_id=$1 AND path=$2", libID, path) - return err -} - -func (s *Store) DeleteBook(ctx context.Context, id int64) error { - _, err := s.P.Exec(ctx, "DELETE FROM books WHERE id=$1", id) - return err -} - -func (s *Store) SetBookState(ctx context.Context, id int64, state, msg string) error { - _, err := s.P.Exec(ctx, "UPDATE books SET state=$2, error_msg=$3 WHERE id=$1", id, state, msg) - return err -} - -func (s *Store) ListBookIDs(ctx context.Context) ([]int64, error) { - rows, err := s.P.Query(ctx, "SELECT id FROM books") - if err != nil { - return nil, err - } - defer rows.Close() - var out []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - out = append(out, id) - } - return out, rows.Err() -} - -func (s *Store) ListBooks(ctx context.Context, libID int64, q, prefix string, userID int64) ([]BookView, error) { - rows, err := s.P.Query(ctx, - `SELECT b.id, b.library_id, b.path, b.title, b.format, b.file_size, b.mod_ts, - b.page_count, b.state, b.error_msg, b.added_at, l.name, COALESCE(p.percent, 0) - FROM books b JOIN libraries l ON l.id = b.library_id - LEFT JOIN reading_progress p ON p.user_id = $4 AND p.library_id = b.library_id AND p.book_path = b.path - WHERE ($1 = 0 OR b.library_id = $1) - AND ($2 = '' OR lower(b.title) LIKE '%' || lower($2) || '%') - AND ($3 = '' OR b.path LIKE $3 || '%') - ORDER BY l.name, b.path`, libID, q, prefix, userID) - if err != nil { - return nil, err - } - defer rows.Close() - var out []BookView - for rows.Next() { - var v BookView - err := rows.Scan(&v.ID, &v.LibraryID, &v.Path, &v.Title, &v.Format, - &v.FileSize, &v.ModTS, &v.PageCount, &v.State, &v.ErrMsg, &v.AddedAt, - &v.LibraryName, &v.Percent) - if err != nil { - return nil, err - } - out = append(out, v) - } - return out, rows.Err() -} - -func (s *Store) BookHashes(ctx context.Context) (map[int64][2]int64, error) { - rows, err := s.P.Query(ctx, "SELECT id, file_size, mod_ts FROM books") - if err != nil { - return nil, err - } - defer rows.Close() - out := map[int64][2]int64{} - for rows.Next() { - var id int64 - var v [2]int64 - if err := rows.Scan(&id, &v[0], &v[1]); err != nil { - return nil, err - } - out[id] = v - } - return out, rows.Err() -} - -// ---------- progress ---------- - -func (s *Store) UpsertProgress(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64) error { - _, err := s.P.Exec(ctx, - `INSERT INTO reading_progress (user_id, library_id, book_path, locator, percent, updated_at) - VALUES ($1,$2,$3,$4,$5,now()) - ON CONFLICT (user_id, library_id, book_path) - DO UPDATE SET locator=$4, percent=$5, updated_at=now()`, - userID, libID, bookPath, locator, percent) - return err -} - -func (s *Store) ListProgress(ctx context.Context, userID int64) ([]Progress, error) { - rows, err := s.P.Query(ctx, - `SELECT p.library_id, l.name, p.book_path, COALESCE(b.title, ''), p.locator, p.percent, p.updated_at - FROM reading_progress p JOIN libraries l ON l.id = p.library_id - LEFT JOIN books b ON b.library_id = p.library_id AND b.path = p.book_path - WHERE p.user_id = $1 ORDER BY p.updated_at DESC`, userID) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Progress - for rows.Next() { - var pr Progress - if err := rows.Scan(&pr.LibraryID, &pr.LibraryName, &pr.BookPath, &pr.Title, - &pr.Locator, &pr.Percent, &pr.UpdatedAt); err != nil { - return nil, err - } - out = append(out, pr) - } - return out, rows.Err() -} - -// ---------- bookmarks ---------- - type Bookmark struct { ID int64 LibraryID int64 @@ -326,51 +88,6 @@ type Bookmark struct { CreatedAt time.Time } -func (s *Store) InsertBookmark(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64, note string) (int64, error) { - var id int64 - err := s.P.QueryRow(ctx, - `INSERT INTO bookmarks (user_id, library_id, book_path, locator, percent, note) - VALUES ($1,$2,$3,$4,$5,$6) RETURNING id`, - userID, libID, bookPath, locator, percent, note).Scan(&id) - return id, err -} - -func (s *Store) ListBookmarks(ctx context.Context, userID, libID int64, bookPath string) ([]Bookmark, error) { - rows, err := s.P.Query(ctx, - `SELECT id, library_id, book_path, locator, percent, note, created_at - FROM bookmarks WHERE user_id=$1 AND library_id=$2 AND book_path=$3 - ORDER BY percent ASC, id ASC`, userID, libID, bookPath) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Bookmark - for rows.Next() { - var b Bookmark - if err := rows.Scan(&b.ID, &b.LibraryID, &b.BookPath, &b.Locator, &b.Percent, &b.Note, &b.CreatedAt); err != nil { - return nil, err - } - out = append(out, b) - } - return out, rows.Err() -} - -func (s *Store) UpdateBookmarkNote(ctx context.Context, userID, id int64, note string) (bool, error) { - res, err := s.P.Exec(ctx, `UPDATE bookmarks SET note=$3 WHERE id=$1 AND user_id=$2`, id, userID, note) - return res.RowsAffected() > 0, err -} - -func (s *Store) DeleteBookmark(ctx context.Context, userID, id int64) (bool, error) { - res, err := s.P.Exec(ctx, `DELETE FROM bookmarks WHERE id=$1 AND user_id=$2`, id, userID) - return res.RowsAffected() > 0, err -} - -func (s *Store) GetProgress(ctx context.Context, userID, libID int64, bookPath string) (Progress, error) { - var pr Progress - err := s.P.QueryRow(ctx, - `SELECT library_id, book_path, locator, percent, updated_at - FROM reading_progress WHERE user_id=$1 AND library_id=$2 AND book_path=$3`, - userID, libID, bookPath). - Scan(&pr.LibraryID, &pr.BookPath, &pr.Locator, &pr.Percent, &pr.UpdatedAt) - return pr, err -} +// Pool returns the underlying pool for test setup only. +// Production code should use Store methods exclusively. +func (s *Store) Pool() *pgxpool.Pool { return s.p } diff --git a/backend/internal/store/store_test.go b/backend/internal/store/store_test.go index b91dda5..0eb8921 100644 --- a/backend/internal/store/store_test.go +++ b/backend/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "os" "testing" @@ -129,3 +130,50 @@ func TestProgressUpsertAndJoin(t *testing.T) { t.Fatalf("views %+v", views) } } + +func TestDeleteUserLastAdmin(t *testing.T) { + s := setup(t) + ctx := context.Background() + id, err := s.CreateUser(ctx, "onlyadmin", "hash", "admin") + if err != nil { + t.Fatal(err) + } + err = s.DeleteUser(ctx, id) + if !errors.Is(err, ErrLastAdmin) { + t.Fatalf("expected ErrLastAdmin, got %v", err) + } + // Verify user still exists. + _, err = s.GetUserByID(ctx, id) + if err != nil { + t.Fatal("admin should still exist after blocked delete") + } +} + +func TestDeleteUserNonLastAdmin(t *testing.T) { + s := setup(t) + ctx := context.Background() + id1, _ := s.CreateUser(ctx, "admin1", "h", "admin") + _, _ = s.CreateUser(ctx, "admin2", "h", "admin") + if err := s.DeleteUser(ctx, id1); err != nil { + t.Fatalf("should allow deleting non-last admin: %v", err) + } +} + +func TestDeleteUserMember(t *testing.T) { + s := setup(t) + ctx := context.Background() + id, _ := s.CreateUser(ctx, "member", "h", "member") + if err := s.DeleteUser(ctx, id); err != nil { + t.Fatalf("member delete should succeed: %v", err) + } +} + +func TestIsUniqueViolation(t *testing.T) { + s := setup(t) + ctx := context.Background() + _, _ = s.CreateUser(ctx, "dup", "h", "member") + _, err := s.CreateUser(ctx, "dup", "h", "member") + if !IsUniqueViolation(err) { + t.Fatalf("expected unique violation, got %v", err) + } +} diff --git a/backend/internal/store/users.go b/backend/internal/store/users.go new file mode 100644 index 0000000..12f1725 --- /dev/null +++ b/backend/internal/store/users.go @@ -0,0 +1,81 @@ +package store + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +const userCols = "id, username, password_hash, role, created_at" + +func (s *Store) CountUsers(ctx context.Context) (int, error) { + var n int + err := s.p.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&n) + return n, err +} + +func (s *Store) CreateUser(ctx context.Context, username, hash, role string) (int64, error) { + var id int64 + err := s.p.QueryRow(ctx, + "INSERT INTO users (username, password_hash, role) VALUES ($1,$2,$3) RETURNING id", + username, hash, role).Scan(&id) + return id, err +} + +func (s *Store) GetUserByName(ctx context.Context, username string) (User, error) { + return scanUser(s.p.QueryRow(ctx, "SELECT "+userCols+" FROM users WHERE username=$1", username)) +} + +func (s *Store) GetUserByID(ctx context.Context, id int64) (User, error) { + return scanUser(s.p.QueryRow(ctx, "SELECT "+userCols+" FROM users WHERE id=$1", id)) +} + +func (s *Store) ListUsers(ctx context.Context) ([]User, error) { + rows, err := s.p.Query(ctx, "SELECT "+userCols+" FROM users ORDER BY id") + if err != nil { + return nil, err + } + defer rows.Close() + var out []User + for rows.Next() { + var u User + if err := rows.Scan(&u.ID, &u.Username, &u.PasswordHash, &u.Role, &u.CreatedAt); err != nil { + return nil, err + } + out = append(out, u) + } + return out, rows.Err() +} + +// DeleteUser deletes a user with a transactional last-admin check (B5). +// Returns ErrLastAdmin if the target is the last admin. +func (s *Store) DeleteUser(ctx context.Context, id int64) error { + tx, err := s.p.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + var role string + if err := tx.QueryRow(ctx, "SELECT role FROM users WHERE id=$1 FOR UPDATE", id).Scan(&role); err != nil { + return err // includes pgx.ErrNoRows + } + if role == "admin" { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM users WHERE role='admin'").Scan(&n); err != nil { + return err + } + if n <= 1 { + return ErrLastAdmin + } + } + if _, err := tx.Exec(ctx, "DELETE FROM users WHERE id=$1", id); err != nil { + return err + } + return tx.Commit(ctx) +} + +func scanUser(row pgx.Row) (User, error) { + var u User + err := row.Scan(&u.ID, &u.Username, &u.PasswordHash, &u.Role, &u.CreatedAt) + return u, err +} diff --git a/backend/internal/upload/upload.go b/backend/internal/upload/upload.go new file mode 100644 index 0000000..17abb21 --- /dev/null +++ b/backend/internal/upload/upload.go @@ -0,0 +1,357 @@ +// Package upload owns the resumable chunked-upload subsystem and the shared +// unique-path placement helper used by both chunked and single-file uploads. +// +// A session lives at /.uploads// (meta.json + parts/N). The uid +// is a fingerprint of (libID, name, size, chunkSize), so re-initialising the +// same file resumes the existing session instead of restarting it. Expired +// sessions are swept by the scanner ticker (B16), not on the request path. +// +// This package is HTTP-free: it returns sentinel errors that the handlers map +// onto status/code/message tuples. The prior client-facing contract is +// preserved exactly. +package upload + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "booklib/internal/bookfile" + "booklib/internal/ports" +) + +const ( + maxChunkBytes = 32 << 20 + defaultChunk = 8 << 20 + uploadSessTTL = 24 * time.Hour + uploadSessionIn = ".uploads" +) + +// Sentinel errors returned to handlers for status/code mapping. The four shared +// with ports (ErrTooLarge/ErrNotFound/ErrIncomplete/ErrSizeMismatch) live in the +// ports package so the interface contract and the implementation agree. +var ( + ErrBadName = errors.New("bad name") + ErrBadFormat = errors.New("bad format") + ErrBadSize = errors.New("bad size") + ErrBadChunk = errors.New("bad chunk size") + ErrBadUploadID = errors.New("bad upload id") + ErrBadIndex = errors.New("bad part index") + ErrPartTooBig = errors.New("part exceeds declared size") + ErrPartSizeMismatch = errors.New("part size mismatch") + ErrCorrupt = errors.New("corrupt session") +) + +// OpError wraps an internal filesystem/IO failure with the short operation label +// that the client-facing 500 message uses, preserving the prior contract strings +// ("create session", "write meta", "create part", ...). +type OpError struct { + Op string + Err error +} + +func (e *OpError) Error() string { return e.Op + ": " + e.Err.Error() } +func (e *OpError) Unwrap() error { return e.Err } + +func opErr(op string, err error) error { return &OpError{Op: op, Err: err} } + +type uploadMeta struct { + Name string `json:"name"` + Size int64 `json:"size"` + ChunkSize int64 `json:"chunkSize"` + LibraryID int64 `json:"libraryId"` +} + +// U is the upload subsystem. It is stateless beyond the filesystem session dir. +type U struct { + booksDir string + uploadMaxMB int64 +} + +// New builds the upload subsystem. booksDir is the storage root (sessions live +// under booksDir/.uploads); uploadMaxMB caps the total declared file size. +func New(booksDir string, uploadMaxMB int64) *U { + return &U{booksDir: booksDir, uploadMaxMB: uploadMaxMB} +} + +// compile-time proof that *U satisfies the consumer-side interface. +var _ ports.UploadSessions = (*U)(nil) + +// ---------- pure helpers ---------- + +func validUploadID(s string) bool { + if len(s) != 32 { + return false + } + for _, r := range s { + if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) { + return false + } + } + return true +} + +func uploadIDFor(libID int64, name string, size, chunk int64) string { + h := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%d|%d", libID, name, size, chunk))) + return hex.EncodeToString(h[:16]) +} + +func (u *U) uploadDir(uid string) string { + return filepath.Join(filepath.Clean(u.booksDir), uploadSessionIn, uid) +} + +func chunkRange(m uploadMeta, i int64) (int64, int64) { + lo := i * m.ChunkSize + hi := min(lo+m.ChunkSize, m.Size) + return lo, hi +} + +func numParts(m uploadMeta) int64 { + return (m.Size + m.ChunkSize - 1) / m.ChunkSize +} + +// ---------- session meta ---------- + +// loadMeta validates uid + reads meta.json. Returns ErrBadUploadID, +// ports.ErrNotFound, or ErrCorrupt on failure. +func (u *U) loadMeta(uid string) (uploadMeta, string, error) { + if !validUploadID(uid) { + return uploadMeta{}, "", ErrBadUploadID + } + dir := u.uploadDir(uid) + b, e := os.ReadFile(filepath.Join(dir, "meta.json")) + if e != nil { + return uploadMeta{}, "", ports.ErrNotFound + } + var m uploadMeta + if json.Unmarshal(b, &m) != nil { + return uploadMeta{}, "", ErrCorrupt + } + return m, dir, nil +} + +// LibraryID returns the target library id recorded in the session, so the +// handler can resolve+validate the library root before calling Complete. +func (u *U) LibraryID(ctx context.Context, uid string) (int64, error) { + m, _, e := u.loadMeta(uid) + if e != nil { + return 0, e + } + return m.LibraryID, nil +} + +// ---------- public API (satisfies ports.UploadSessions) ---------- + +// Init validates the declared upload, then creates or resumes a session. The +// returned uid is deterministic for a given (libID, safeName, size, chunkSize), +// so a re-init of the same file resumes; a fingerprint collision with different +// content restarts the session. +func (u *U) Init(ctx context.Context, libID int64, name string, size, chunkSize int64) (string, error) { + safe := bookfile.SafeName(name) + if safe == "" { + return "", ErrBadName + } + if bookfile.FormatFromExt(safe) == "" { + return "", ErrBadFormat + } + if size <= 0 { + return "", ErrBadSize + } + if size > u.uploadMaxMB<<20 { + return "", ports.ErrTooLarge + } + if chunkSize == 0 { + chunkSize = defaultChunk + } + if chunkSize > maxChunkBytes { + return "", ErrBadChunk + } + uid := uploadIDFor(libID, safe, size, chunkSize) + dir := u.uploadDir(uid) + meta := uploadMeta{Name: safe, Size: size, ChunkSize: chunkSize, LibraryID: libID} + if b, e := os.ReadFile(filepath.Join(dir, "meta.json")); e == nil { + var old uploadMeta + if json.Unmarshal(b, &old) == nil && old == meta { // same fingerprint → resume + return uid, nil + } + os.RemoveAll(dir) // fingerprint collided but content differs → restart + } + if e := os.MkdirAll(filepath.Join(dir, "parts"), 0o755); e != nil { + return "", opErr("create session", e) + } + b, _ := json.Marshal(meta) + if e := os.WriteFile(filepath.Join(dir, "meta.json"), b, 0o644); e != nil { + return "", opErr("write meta", e) + } + return uid, nil +} + +// Status returns the sorted indices of parts already received. +func (u *U) Status(ctx context.Context, uid string) ([]int64, error) { + _, dir, e := u.loadMeta(uid) + if e != nil { + return nil, e + } + recv := []int64{} + es, e := os.ReadDir(filepath.Join(dir, "parts")) + if e == nil { + for _, en := range es { + if i, e := strconv.ParseInt(en.Name(), 10, 64); e == nil { + recv = append(recv, i) + } + } + } + sort.Slice(recv, func(i, j int) bool { return recv[i] < recv[j] }) + return recv, nil +} + +// PutPart writes one part via tmp+rename (B4: a truncated part is never reported +// as received). body is read up to the declared part size; reading past it yields +// ErrPartTooBig, a short read yields ErrPartSizeMismatch. maxSize, when > 0, is a +// defensive ceiling on bytes read. +func (u *U) PutPart(ctx context.Context, uid string, index int64, body io.Reader, maxSize int64) error { + m, dir, e := u.loadMeta(uid) + if e != nil { + return e + } + if index < 0 || index >= numParts(m) { + return ErrBadIndex + } + lo, hi := chunkRange(m, index) + want := hi - lo + limit := want + 1 + if maxSize > 0 && maxSize+1 < limit { + limit = maxSize + 1 + } + p := filepath.Join(dir, "parts", strconv.FormatInt(index, 10)) + tmp := p + ".tmp" + f, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if e != nil { + return opErr("create part", e) + } + n, copyErr := io.Copy(f, io.LimitReader(body, limit)) + f.Close() + if copyErr != nil { + os.Remove(tmp) + return ErrPartSizeMismatch + } + if n > want { + os.Remove(tmp) + return ErrPartTooBig + } + if n != want { + os.Remove(tmp) + return ErrPartSizeMismatch + } + if e := os.Rename(tmp, p); e != nil { + os.Remove(tmp) + return opErr("rename part", e) + } + return nil +} + +// Complete verifies every part is present and correctly sized, assembles them +// into a tmp file, then atomically renames it into root under a unique name. +// It returns the path relative to root. The session dir is removed on success. +func (u *U) Complete(ctx context.Context, uid, root string) (string, error) { + m, dir, e := u.loadMeta(uid) + if e != nil { + return "", e + } + var total int64 + for i := int64(0); i < numParts(m); i++ { + lo, hi := chunkRange(m, i) + fi, e := os.Stat(filepath.Join(dir, "parts", strconv.FormatInt(i, 10))) + if e != nil || fi.Size() != hi-lo { + return "", ports.ErrIncomplete + } + total += fi.Size() + } + if total != m.Size { + return "", ports.ErrSizeMismatch + } + dst, e := u.UniquePath(root, m.Name) + if e != nil { + return "", e // os.ErrInvalid / os.ErrExist → handler maps to 403 + } + tmp := filepath.Join(dir, "assembled") + out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if e != nil { + return "", opErr("create tmp", e) + } + for i := int64(0); i < numParts(m); i++ { + pf, e := os.Open(filepath.Join(dir, "parts", strconv.FormatInt(i, 10))) + if e != nil { + out.Close() + return "", opErr("open part", e) + } + _, copyErr := io.Copy(out, pf) + pf.Close() + if copyErr != nil { + out.Close() + os.Remove(tmp) + return "", opErr("assemble", copyErr) + } + } + out.Close() + if e := os.Rename(tmp, dst); e != nil { // atomic placement; scanner picks it up + os.Remove(tmp) + return "", opErr("rename", e) + } + os.RemoveAll(dir) + return strings.TrimPrefix(dst, root+string(os.PathSeparator)), nil +} + +// Sweep removes session dirs untouched for longer than the TTL. Best-effort: +// a missing/unreadable base dir is not an error. Run from the scanner ticker +// (B16) instead of the request path. +func (u *U) Sweep(ctx context.Context) error { + base := filepath.Join(filepath.Clean(u.booksDir), uploadSessionIn) + es, e := os.ReadDir(base) + if e != nil { + return nil // no sessions yet + } + for _, en := range es { + if fi, e := en.Info(); e == nil && time.Since(fi.ModTime()) > uploadSessTTL { + os.RemoveAll(filepath.Join(base, en.Name())) + } + } + return nil +} + +// UniquePath returns a path under root for name that does not yet exist, +// appending " (n)" on collision. The cleaned name must stay inside root. +// Shared by single-file and chunked upload completion. +func (u *U) UniquePath(root, name string) (string, error) { + ext := filepath.Ext(name) + base := strings.TrimSuffix(name, ext) + for i := 0; ; i++ { + cand := base + ext + if i > 0 { + cand = base + " (" + strconv.Itoa(i) + ")" + ext + } + p := filepath.Join(root, cand) + if filepath.Clean(p) != filepath.Join(root, filepath.Clean(cand)) || + !strings.HasPrefix(filepath.Clean(p), root+string(os.PathSeparator)) { + return "", os.ErrInvalid + } + if _, e := os.Stat(p); os.IsNotExist(e) { + return p, nil + } else if e != nil { + return "", e + } + if i > 999 { + return "", os.ErrExist + } + } +} diff --git a/backend/internal/upload/upload_test.go b/backend/internal/upload/upload_test.go new file mode 100644 index 0000000..e8fbe99 --- /dev/null +++ b/backend/internal/upload/upload_test.go @@ -0,0 +1,252 @@ +package upload + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "booklib/internal/ports" +) + +func newU(t *testing.T) *U { + t.Helper() + dir := t.TempDir() + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + return New(resolved, 1) // 1MB cap, mirrors handler test cfg +} + +func mustInit(t *testing.T, u *U, libID int64, name string, size, chunk int64) string { + t.Helper() + uid, err := u.Init(context.Background(), libID, name, size, chunk) + if err != nil { + t.Fatalf("Init(%q): %v", name, err) + } + return uid +} + +func TestUploadIDForDeterministic(t *testing.T) { + a := uploadIDFor(1, "x.cbz", 100, 50) + b := uploadIDFor(1, "x.cbz", 100, 50) + c := uploadIDFor(2, "x.cbz", 100, 50) + if a != b { + t.Fatalf("same fingerprint must yield same uid: %s vs %s", a, b) + } + if a == c { + t.Fatal("different libID must yield different uid") + } + if !validUploadID(a) || validUploadID("zzz") || validUploadID(strings.Repeat("a", 31)) { + t.Fatal("validUploadID broken") + } +} + +func TestInitValidation(t *testing.T) { + u := newU(t) + ctx := context.Background() + if _, err := u.Init(ctx, 1, "", 100, 0); !errors.Is(err, ErrBadName) { + t.Fatalf("empty name: want ErrBadName got %v", err) + } + if _, err := u.Init(ctx, 1, "virus.exe", 100, 0); !errors.Is(err, ErrBadFormat) { + t.Fatalf("bad ext: want ErrBadFormat got %v", err) + } + if _, err := u.Init(ctx, 1, "ok.cbz", 0, 0); !errors.Is(err, ErrBadSize) { + t.Fatalf("zero size: want ErrBadSize got %v", err) + } + if _, err := u.Init(ctx, 1, "big.cbz", 2<<20, 0); !errors.Is(err, ports.ErrTooLarge) { + t.Fatalf("oversize: want ErrTooLarge got %v", err) + } + if _, err := u.Init(ctx, 1, "ok.cbz", 100, 40<<20); !errors.Is(err, ErrBadChunk) { + t.Fatalf("huge chunk: want ErrBadChunk got %v", err) + } + // 默认 chunk 生效且 uid 稳定 + uid := mustInit(t, u, 1, "ok.cbz", 100, 0) + if uid != uploadIDFor(1, "ok.cbz", 100, defaultChunk) { + t.Fatal("chunkSize=0 must default to defaultChunk in fingerprint") + } + // SafeName 清洗:path 形式取 base + if uid2 := mustInit(t, u, 1, `C:\dir\book.cbz`, 100, 0); uid2 != uploadIDFor(1, "book.cbz", 100, defaultChunk) { + t.Fatalf("windows path must be sanitized to base name, got uid %s", uid2) + } +} + +func TestInitResumeAndRestart(t *testing.T) { + u := newU(t) + ctx := context.Background() + uid := mustInit(t, u, 1, "r.cbz", 900000, 400000) + if err := u.PutPart(ctx, uid, 0, bytes.NewReader(bytes.Repeat([]byte("x"), 400000)), 0); err != nil { + t.Fatalf("PutPart: %v", err) + } + // 同指纹 re-init → 复用会话,分片保留 + uid2, err := u.Init(ctx, 1, "r.cbz", 900000, 400000) + if err != nil || uid2 != uid { + t.Fatalf("resume: want same uid, got %s err=%v", uid2, err) + } + recv, err := u.Status(ctx, uid) + if err != nil || len(recv) != 1 || recv[0] != 0 { + t.Fatalf("resume must keep parts: %v err=%v", recv, err) + } + // 同指纹位但内容不同(size 变)→ 新会话,旧目录被换掉是安全的(uid 不同) + uid3 := mustInit(t, u, 1, "r.cbz", 800000, 400000) + if uid3 == uid { + t.Fatal("different size must yield different uid") + } +} + +func TestPutPartErrors(t *testing.T) { + u := newU(t) + ctx := context.Background() + uid := mustInit(t, u, 1, "p.cbz", 1000, 400) + if err := u.PutPart(ctx, uid, 9, bytes.NewReader(bytes.Repeat([]byte("y"), 400)), 0); !errors.Is(err, ErrBadIndex) { + t.Fatalf("index oob: want ErrBadIndex got %v", err) + } + if err := u.PutPart(ctx, uid, -1, bytes.NewReader(nil), 0); !errors.Is(err, ErrBadIndex) { + t.Fatalf("negative index: want ErrBadIndex got %v", err) + } + // 超期望体积 → ErrPartTooBig + if err := u.PutPart(ctx, uid, 0, bytes.NewReader(bytes.Repeat([]byte("y"), 500)), 0); !errors.Is(err, ErrPartTooBig) { + t.Fatalf("oversize part: want ErrPartTooBig got %v", err) + } + // 短读 → ErrPartSizeMismatch + if err := u.PutPart(ctx, uid, 0, bytes.NewReader(bytes.Repeat([]byte("y"), 300)), 0); !errors.Is(err, ErrPartSizeMismatch) { + t.Fatalf("short part: want ErrPartSizeMismatch got %v", err) + } + // 失败不留 parts/B4: 截断分片不被 Status 报告为已接收 + recv, err := u.Status(ctx, uid) + if err != nil || len(recv) != 0 { + t.Fatalf("failed parts must not be received: %v err=%v", recv, err) + } + // 未知/非法 uid + if _, err := u.Status(ctx, "deadbeefdeadbeefdeadbeefdeadbeef"); !errors.Is(err, ports.ErrNotFound) { + t.Fatalf("unknown uid: want ErrNotFound got %v", err) + } + if _, err := u.Status(ctx, "zzz"); !errors.Is(err, ErrBadUploadID) { + t.Fatalf("bad uid: want ErrBadUploadID got %v", err) + } +} + +func TestCompleteHappyPathAndCleanup(t *testing.T) { + u := newU(t) + ctx := context.Background() + root := filepath.Join(u.booksDir, "lib") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + content := bytes.Repeat([]byte("調教開關第二季!"), 300) // ~6.3KB multibyte + size := int64(len(content)) + const chunk = int64(2000) + uid := mustInit(t, u, 7, "調教開關:第二季.zip", size, chunk) + n := int((size + chunk - 1) / chunk) + // 乱序上传:索引顺序 1,2,...,n-1,0 —— 覆盖所有分片且非递增 + for step := 1; step <= n; step++ { + idx := int64(step % n) + lo := idx * chunk + hi := min(lo+chunk, size) + if err := u.PutPart(ctx, uid, idx, bytes.NewReader(content[lo:hi]), 0); err != nil { + t.Fatalf("PutPart %d: %v", idx, err) + } + } + // 缺片 → ErrIncomplete(先删一片验证) + part0 := filepath.Join(u.uploadDir(uid), "parts", "0") + saved, _ := os.ReadFile(part0) + os.Remove(part0) + if _, err := u.Complete(ctx, uid, root); !errors.Is(err, ports.ErrIncomplete) { + t.Fatalf("missing part: want ErrIncomplete got %v", err) + } + os.WriteFile(part0, saved, 0o644) + + rel, err := u.Complete(ctx, uid, root) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if rel != "調教開關:第二季.zip" { + t.Fatalf("rel path: %q", rel) + } + got, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil || !bytes.Equal(got, content) { + t.Fatalf("assembled content wrong: err=%v", err) + } + if _, err := os.Stat(u.uploadDir(uid)); !os.IsNotExist(err) { + t.Fatal("session dir must be removed after complete") + } +} + +func TestCompleteUniquePathCollision(t *testing.T) { + u := newU(t) + ctx := context.Background() + root := filepath.Join(u.booksDir, "lib") + os.MkdirAll(root, 0o755) + os.WriteFile(filepath.Join(root, "dup.cbz"), []byte("existing"), 0o644) + uid := mustInit(t, u, 1, "dup.cbz", 4, 4) + if err := u.PutPart(ctx, uid, 0, bytes.NewReader([]byte("new!")), 0); err != nil { + t.Fatal(err) + } + rel, err := u.Complete(ctx, uid, root) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if rel != "dup (1).cbz" { + t.Fatalf("collision must add suffix, got %q", rel) + } +} + +func TestUniquePathTraversalRejected(t *testing.T) { + u := newU(t) + root := filepath.Join(u.booksDir, "lib") + if _, err := u.UniquePath(root, "../escape.cbz"); !errors.Is(err, os.ErrInvalid) { + t.Fatalf("traversal: want os.ErrInvalid got %v", err) + } + p, err := u.UniquePath(root, "ok.cbz") + if err != nil || p != filepath.Join(root, "ok.cbz") { + t.Fatalf("valid name: %q %v", p, err) + } +} + +func TestSweepRemovesOnlyExpired(t *testing.T) { + u := newU(t) + ctx := context.Background() + fresh := mustInit(t, u, 1, "fresh.cbz", 10, 10) + stale := mustInit(t, u, 1, "stale.cbz", 10, 10) + // 把 stale 会话 mtime 拨到 TTL 之前 + old := time.Now().Add(-uploadSessTTL - time.Hour) + dir := u.uploadDir(stale) + if err := os.Chtimes(filepath.Join(dir, "meta.json"), old, old); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(dir, old, old); err != nil { + t.Fatal(err) + } + if err := u.Sweep(ctx); err != nil { + t.Fatalf("Sweep: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatal("stale session must be swept") + } + if _, err := os.Stat(u.uploadDir(fresh)); err != nil { + t.Fatalf("fresh session must survive: %v", err) + } + // 空目录/不存在 base 都不报错 + os.RemoveAll(filepath.Join(u.booksDir, uploadSessionIn)) + if err := u.Sweep(ctx); err != nil { + t.Fatalf("Sweep on missing base: %v", err) + } +} + +func TestLibraryID(t *testing.T) { + u := newU(t) + ctx := context.Background() + uid := mustInit(t, u, 42, "lib.cbz", 10, 10) + id, err := u.LibraryID(ctx, uid) + if err != nil || id != 42 { + t.Fatalf("LibraryID: %d %v", id, err) + } + if _, err := u.LibraryID(ctx, "deadbeefdeadbeefdeadbeefdeadbeef"); !errors.Is(err, ports.ErrNotFound) { + t.Fatalf("unknown uid: want ErrNotFound got %v", err) + } +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 80a7f8a..31493cc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,23 +10,78 @@ The format loosely follows Keep a Changelog and can be adapted to the team's hab ### Added / 新增 +- API: resumable chunked upload protocol for large files — `POST /api/libraries/:id/upload/init` (fingerprint-derived deterministic `uploadId`, rejects totals over `UPLOAD_MAX_MB` with `413 too_large`), `PUT /api/uploads/:uid/parts/:index` (parts ≤ 32MB), `GET /api/uploads/:uid` (received parts, for resume), `POST /api/uploads/:uid/complete` (assemble + atomic land, same path contract as single-POST upload). Sessions persist under `BOOKS_DIR/.uploads/` with a periodic sweep. `UPLOAD_MAX_MB` is now wired through both compose stacks/`.env`; `.env.example` sets 2048 and drops `NGINX_CLIENT_MAX_BODY_SIZE` to 32m (nginx only ever sees one chunk). +- API:新增大文件可续传分片上传协议——`POST /api/libraries/:id/upload/init`(按指纹派生确定性 `uploadId`,总量超 `UPLOAD_MAX_MB` 返回 `413 too_large`)、`PUT /api/uploads/:uid/parts/:index`(单片 ≤32MB)、`GET /api/uploads/:uid`(查询已传分片以续传)、`POST /api/uploads/:uid/complete`(拼接后原子落盘,返回与单发上传一致的 `path`)。会话存于 `BOOKS_DIR/.uploads/`,定期清理。`UPLOAD_MAX_MB` 已接入两份 compose/`.env`;`.env.example` 调至 2048 并将 `NGINX_CLIENT_MAX_BODY_SIZE` 降为 32m(nginx 只见单个分片)。 + - API: per-user bookmarks — `GET/POST /api/books/:id/bookmarks` (locator+percent snapshot with optional ≤500-char note; list ordered by percent) and `PATCH/DELETE /api/bookmarks/:id`; not-yours uniformly 404. New `bookmarks` table keyed like progress, cleaned up with the user (no cascade on book delete, same precedent). - API:新增按用户隔离的书签——`GET/POST /api/books/:id/bookmarks`(存当前 locator+percent,备注可选、≤500 字,列表按进度升序)与 `PATCH/DELETE /api/bookmarks/:id`;不属于自己的一律 404。新 `bookmarks` 表与进度同款定位键,随用户删除而清(删书不级联,沿用既有先例)。 - API: CBZ page indexing now skips macOS packaging junk (`__MACOSX/…` and `._*` AppleDouble files), which used to land in the page list as ~163-byte black "pages"; `GET /api/books/:id/pages` additionally returns `chapters:[{title,start}]` derived from the archive's folder structure (e.g. 第1話…), so per-folder comics expose their real organization. - API:CBZ 页索引现会跳过 macOS 打包垃圾(`__MACOSX/…` 与 `._*` 资源叉文件),此前它们以 ~163 字节黑页混入页列表;`GET /api/books/:id/pages` 新增 `chapters:[{title,start}]`,按压缩包内目录结构(如 第1話…)给出真实章节。 -- API: resumable chunked upload protocol for large files — `POST /api/libraries/:id/upload/init` (fingerprint-derived deterministic `uploadId`, rejects totals over `UPLOAD_MAX_MB` with `413 too_large`), `PUT /api/uploads/:uid/parts/:index` (parts ≤ 32MB), `GET /api/uploads/:uid` (received parts, for resume), `POST /api/uploads/:uid/complete` (assemble + atomic land, same path contract as single-POST upload). Sessions persist under `BOOKS_DIR/.uploads/` with 24h opportunistic sweep. `UPLOAD_MAX_MB` is now wired through both compose stacks/`.env`; `.env.example` sets 2048 and drops `NGINX_CLIENT_MAX_BODY_SIZE` to 32m (nginx only ever sees one chunk). -- API: 新增大文件可续传分片上传协议——`POST /api/libraries/:id/upload/init`(按指纹派生确定性 `uploadId`,总量超 `UPLOAD_MAX_MB` 返回 `413 too_large`)、`PUT /api/uploads/:uid/parts/:index`(单片 ≤32MB)、`GET /api/uploads/:uid`(查询已传分片以续传)、`POST /api/uploads/:uid/complete`(拼接后原子落盘,返回与单发上传一致的 `path`)。会话存于 `BOOKS_DIR/.uploads/`,超 24h 顺手清理。`UPLOAD_MAX_MB` 已接入两份 compose/`.env`;`.env.example` 调至 2048 并将 `NGINX_CLIENT_MAX_BODY_SIZE` 降为 32m(nginx 只见单个分片)。 +- Tests: router contract test (`TestRouterContract`) pins the full route table — any added, removed or renamed route fails the test until the expectation is updated deliberately. +- 测试:新增路由契约测试(`TestRouterContract`),锁定全量路由表——任何路由的增删改名都会使该测试失败,必须显式更新预期。 + +- Tests: hand-written in-memory fakes (`internal/ports/portsfake`) cover all port interfaces, enabling handler unit tests with no PG or Redis. Error semantics mirror the real store (`pgx.ErrNoRows`, `store.ErrLastAdmin`, `*pgconn.PgError{Code:23505}`), so the fakes exercise the same 404/409/400 branches as the database. +- 测试:新增手写内存 fake(`internal/ports/portsfake`),覆盖全部 port 接口,使 handler 单测无需 PG/Redis 即可运行。错误语义与真实 store 一致(`pgx.ErrNoRows`、`store.ErrLastAdmin`、`*pgconn.PgError{Code:23505}`),因此 fake 走到的是与真库相同的 404/409/400 分支。 + +- CI workflow (`.github/workflows/ci.yml`) compatible with both GitHub Actions and Gitea Actions. +- CI 工作流(`.github/workflows/ci.yml`),兼容 GitHub Actions 和 Gitea Actions。 ### Changed / 变更 +- Backend restructured around hexagonal ports: HTTP handlers now depend only on small consumer-side interfaces (`internal/ports`) instead of concrete `*store.Store` / `*redispkg.R` / `*scanner.Scanner`. Domain logic moved out of handlers into `internal/media` (cover/page extraction, page index cache, atomic cache writes) and `internal/upload` (chunked session lifecycle). `cmd/webui/main.go` is the composition root; `api.NewRouter` accepts pure interfaces. +- 后端按六边形端口重构:HTTP handler 现在只依赖 `internal/ports` 中的小口径消费端接口,不再直接持有 `*store.Store` / `*redispkg.R` / `*scanner.Scanner` 等具体类型。域逻辑从 handler 下沉到 `internal/media`(封面/页抽取、页索引缓存、缓存原子写)与 `internal/upload`(分片会话生命周期)。`cmd/webui/main.go` 作为装配根,`api.NewRouter` 只收接口。 + +- Upload session sweep moved off the request path onto the scanner's ticker cycle (B16), so `POST /upload/init` no longer pays for a directory walk. +- 上传会话清理从请求路径移到扫描器的定时周期(B16),`POST /upload/init` 不再顺带付出一次目录遍历的开销。 + +- Add ordered migration system with `schema_migrations` tracking and pg advisory lock for safe multi-replica schema evolution. Existing databases are auto-baselined. To change the schema, add a new `NNNN_description.sql` file under `backend/internal/db/migrations/`; never modify an already-applied file. No down migrations — rollback via backup, fix-forward. +- 新增有序迁移系统,通过 `schema_migrations` 表和 pg advisory lock 实现安全的多副本 schema 演进,已有数据库自动基线化。修改 schema 时在 `backend/internal/db/migrations/` 下新增 `NNNN_description.sql`,已应用的文件不可修改。不支持 down 迁移——回滚靠备份,fix-forward。 + - Scanner: an image-list-less archive (`.zip`/`.cbz` with no page images — video packs, document dumps) is now recorded as `state=error` ("no images in archive") instead of registering as an empty CBZ with a blank reader. - 扫描器:不含任何图片条目的 `.zip`/`.cbz`(视频包、文档包)现记录为 `state=error`("no images in archive"),不再注册成空 CBZ 留下一个白板阅读器。 - API: upload over `UPLOAD_MAX_MB` now returns `413 too_large` with the limit in the message; previously the size abort was misreported as `400 bad_request "multipart field 'file' required"`. - API:超过 `UPLOAD_MAX_MB` 的上传现在返回 `413 too_large` 并在消息中带上限额;此前体积超限被误报为 `400 bad_request "multipart field 'file' required"`。 + - API: `POST /api/libraries` now takes only `{name}`; `root_path` is generated server-side as `BOOKS_DIR/` (no client-supplied paths, validated at creation). - API:`POST /api/libraries` 只需 `{name}`;`root_path` 由服务端生成为 `BOOKS_DIR/<清洗后的库名>`(不再接受客户端指定路径,创建时即校验)。 + - Repo structure conformed to `AGENTS.md`: `web/` renamed to `frontend/`; backend HTTP layer moved from `internal/api` to `cmd/webui/{api,handlers}` (`cmd/server` → `cmd/webui`); README/CHANGELOGs relocated under `docs/` (`README_zh.md` added as Chinese mirror); module-level `.gitignore`s added (`backend/`, `deploy/`); stray root `library/` removed (book files live in `deploy/api/storage/`); debug binaries untracked. - Docs: `docs/README.md` is now the English primary; previous Chinese README mirrored to `docs/README_zh.md`. + +### Fixed / 修复 + +- Rate limiter `IncrWindow` uses atomic Lua script for INCR+EXPIRE, preventing permanent IP lockout on EXPIRE failure (B1). +- 限流器 `IncrWindow` 改用 Lua 脚本原子执行 INCR+EXPIRE,防止 EXPIRE 失败导致 IP 永久锁定(B1)。 +- Distributed lock `Lock` handles `rand.Read` failure by degrading to no-lock instead of using a zero token (B2). +- 分布式锁 `Lock` 在 `rand.Read` 失败时降级为无锁模式,而非使用全零 token(B2)。 +- Lock unlock uses `context.WithoutCancel` to survive caller cancellation (B3). +- 锁的解锁改用 `context.WithoutCancel`,在调用方上下文取消后仍能正常释放(B3)。 +- Upload part writes to `.tmp` then renames, preventing truncated parts from being reported as received (B4). +- 分片上传先写 `.tmp` 再 rename,防止崩溃截断的分片被误报为已接收(B4)。 +- `DeleteUser` last-admin check is now transactional, eliminating TOCTOU race (B5). +- `DeleteUser` 的最后管理员检查改为事务内执行,消除 TOCTOU 竞态(B5)。 +- Single-file upload `io.Copy` errors other than `MaxBytesError` return 500 instead of 413 (B6). +- 单文件上传中非 `MaxBytesError` 的 `io.Copy` 错误返回 500 而非 413(B6)。 +- `/auth/me` distinguishes `no rows` (401) from database errors (503) (B7). +- `/auth/me` 区分无记录(401)和数据库错误(503)(B7)。 +- Library creation rejects reserved names (`cache`, `.uploads`) with `400 reserved_name` (B8). +- 创建书库时拒绝保留名(`cache`、`.uploads`),返回 `400 reserved_name`(B8)。 +- Scanner lock auto-renews every TTL/2 during long scans; per-library single-flight prevents concurrent scans (B9). +- 扫描锁每 TTL/2 自动续期;库级 single-flight 防止并发扫描(B9)。 +- Scanner `SetBookState` errors are now logged instead of silently discarded (B10). +- 扫描器 `SetBookState` 的错误现在会记录日志而非静默丢弃(B10)。 +- Cover write errors fully checked; orphan `.tmp` files cleaned only on failure (B11). +- 封面写入错误全部检查;孤儿 `.tmp` 文件仅在失败路径清理(B11)。 +- Upload handler retries on `O_EXCL` collision for concurrent same-name uploads (B12). +- 上传处理器在 `O_EXCL` 冲突时重试,处理并发同名上传(B12)。 +- Bookmark methods check `err` before `RowsAffected` to avoid invalid reads on query failure (B13). +- 书签方法先检查 `err` 再读 `RowsAffected`,避免查询失败时的无效读取(B13)。 +- Serve goroutine `log.Fatalf` replaced with channel-based shutdown to preserve graceful teardown (B14). +- 服务 goroutine 中的 `log.Fatalf` 改为 channel 通知方式,确保优雅关停流程不被绕过(B14)。 +- `DATABASE_URL` is now validated at startup (required, parseable); empty `REDIS_URL` logs a clear "redis disabled" message (B15). +- `DATABASE_URL` 在启动时校验(必填、可解析);空 `REDIS_URL` 记录明确的 "redis disabled" 日志(B15)。 +- `scripts/smoke.sh` aligned with current API contract, removed ignored `root_path` field (B17). +- `scripts/smoke.sh` 对齐当前 API 契约,移除被忽略的 `root_path` 字段(B17)。 diff --git a/docs/README.md b/docs/README.md index 95e5f99..c4f096f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -68,7 +68,38 @@ Book files: move the contents of the old host `./library/` into `deploy/api/stor ## Read this before changing the schema -`db.Migrate` only runs the `CREATE TABLE IF NOT EXISTS` statements of `schema.sql` — column adds/changes **do not take effect** on existing databases. Before any column change, introduce a `schema_migrations` version table + ordered migrations, otherwise old deployments silently run on the old shape. +Schema changes go through the ordered migration system in `backend/internal/db/migrations/`: + +1. Create a new file: `NNNN_description.sql` (four-digit sequence number, lowercase snake_case). +2. Never modify an already-applied migration file — they are immutable. +3. No down migrations: rollback via database backup, fix-forward. +4. Existing databases are auto-baselined on first startup (0001 marked applied without re-running DDL). +5. Migrations run with `pg_advisory_lock` so `--scale api=N` replicas serialize safely. + +Local gate before each batch merge: `go vet ./... && gofmt -l . && go test -p 1 -count=1 ./...` with dev PG+Redis running. + +## Backend structure + +The backend is organized around consumer-side port interfaces (hexagonal style): + +- `cmd/webui` — binary entry point and composition root: `main.go` builds concrete implementations (`store.Store`, `redispkg.R`, `scanner.Scanner`, `media.M`, `upload.U`) and hands them to `api.NewRouter`, which only accepts the port interfaces. +- `cmd/webui/handlers` — HTTP layer: request binding, auth/authz, error → status mapping. No SQL, no archive/file logic. +- `internal/ports` — the small interfaces handlers depend on (`UserStore`, `LibraryStore`, `BookStore`, `ProgressStore`, `BookmarkStore`, `RateLimiter`, `Scanner`, `Media`, `UploadSessions`) plus shared sentinel errors. Interfaces live on the consumer side, implementations satisfy them. +- `internal/ports/portsfake` — hand-written in-memory fakes for every port, with error semantics mirroring the real store (`pgx.ErrNoRows`, `ErrLastAdmin`, PgError 23505). Handler unit tests run against these with no PG/Redis. +- `internal/media` — cover/page extraction, page-index cache (Redis-backed), atomic cache writes. +- `internal/upload` — chunked upload session lifecycle (init/part/status/complete/sweep). +- `internal/store` — all SQL, one place. +- `internal/scanner` — library walk, ingest (add/update/delete in one pass), sweep riding the scan ticker. +- `internal/bookfile` — shared file utilities (`SafeName`, `Contains`, `Hash`, `FormatFromExt`, cache dir layout). + +Testing is two-tiered: integration tests hit a real PG+Redis via the full router (`handlers/*_test.go` with `setupAPI`), unit tests hit the same router with `portsfake` injected (`handlers/*_unit_test.go`). The route table itself is pinned by `TestRouterContract` in `cmd/webui/api`. + +## CI + +- Workflow: `.github/workflows/ci.yml` (standard GitHub Actions syntax, Gitea Actions compatible). +- **Gitea**: register an `act_runner` instance, enable Actions in repo settings. Works out of the box. +- **GitHub**: works out of the box. +- Until a runner is registered, run the local gate manually before merging. ## PWA diff --git a/docs/README_zh.md b/docs/README_zh.md index 902e02d..17a719e 100644 --- a/docs/README_zh.md +++ b/docs/README_zh.md @@ -68,7 +68,38 @@ docker volume rm book-comic-library_pgdata book-comic-library_cache ## 改 schema 前必读 -`db.Migrate` 只执行 `schema.sql` 的 `CREATE TABLE IF NOT EXISTS`——对已存在的库**加列/改列不会生效**。任何列变更之前,必须先引入 `schema_migrations` 版本表 + 有序迁移脚本,否则老部署会静默跑在旧结构上。 +Schema 变更通过 `backend/internal/db/migrations/` 中的有序迁移系统执行: + +1. 新建文件:`NNNN_description.sql`(四位序号,小写下划线命名)。 +2. 已应用的迁移文件不可修改——它们是不可变的。 +3. 不支持 down 迁移:回滚靠数据库备份,fix-forward。 +4. 已有数据库在首次启动时自动基线化(0001 标记为已应用,不重跑 DDL)。 +5. 迁移使用 `pg_advisory_lock` 确保 `--scale api=N` 副本串行执行。 + +每批合入前的本地门禁:`go vet ./... && gofmt -l . && go test -p 1 -count=1 ./...`(需启动 dev PG+Redis)。 + +## 后端结构 + +后端按消费端接口组织(六边形风格): + +- `cmd/webui` —— 二进制入口与装配根:`main.go` 构造具体实现(`store.Store`、`redispkg.R`、`scanner.Scanner`、`media.M`、`upload.U`)并交给 `api.NewRouter`,后者只接受 port 接口。 +- `cmd/webui/handlers` —— HTTP 层:参数绑定、认证/鉴权、错误→状态码映射。没有 SQL,没有压缩包/文件逻辑。 +- `internal/ports` —— handlers 依赖的小口径接口(`UserStore`、`LibraryStore`、`BookStore`、`ProgressStore`、`BookmarkStore`、`RateLimiter`、`Scanner`、`Media`、`UploadSessions`)与共享 sentinel 错误。接口定义在消费端,实现方来满足它们。 +- `internal/ports/portsfake` —— 全部 port 的手写内存 fake,错误语义与真实 store 一致(`pgx.ErrNoRows`、`ErrLastAdmin`、PgError 23505)。handler 单测无需 PG/Redis。 +- `internal/media` —— 封面/页抽取、页索引缓存(Redis)、缓存原子写。 +- `internal/upload` —— 分片上传会话生命周期(init/part/status/complete/sweep)。 +- `internal/store` —— 所有 SQL,集中一处。 +- `internal/scanner` —— 书库遍历、ingest(增/改/删一趟完成)、会话清扫搭扫描 ticker 顺风车。 +- `internal/bookfile` —— 共享文件工具(`SafeName`、`Contains`、`Hash`、`FormatFromExt`、缓存目录布局)。 + +测试分两层:集成测试走真实 PG+Redis、过完整 router(`handlers/*_test.go` 的 `setupAPI`);单测注入 `portsfake`、过同一个 router(`handlers/*_unit_test.go`)。路由表本身由 `cmd/webui/api` 的 `TestRouterContract` 钉死。 + +## CI + +- 工作流:`.github/workflows/ci.yml`(标准 GitHub Actions 语法,兼容 Gitea Actions)。 +- **Gitea**:注册 `act_runner` 实例,在仓库设置中启用 Actions,开箱即用。 +- **GitHub**:开箱即用。 +- Runner 注册前,合入前手动执行本地门禁。 ## PWA diff --git a/docs/superpowers/plans/2026-09-14-backend-hardening.md b/docs/superpowers/plans/2026-09-14-backend-hardening.md new file mode 100644 index 0000000..af1876e --- /dev/null +++ b/docs/superpowers/plans/2026-09-14-backend-hardening.md @@ -0,0 +1,1885 @@ +# Backend Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make schema changes safe via ordered migrations, fix 17 confirmed defects, restructure business logic into `internal/` with port interfaces, and introduce a CI workflow. + +**Architecture:** Three sequential batches. Batch A builds the migration system and CI foundation. Batch B fixes bugs B1–B13 with reproduction tests. Batch C performs structural reorganization (ports/store split/media/upload/scanner) with contract tests and fake-based unit tests. Each batch keeps tests green throughout. + +**Tech Stack:** Go 1.26, gin, pgx/v5, go-redis/v9, PostgreSQL 16, Redis 7, GitHub Actions (Gitea Actions compatible), embedded SQL migrations with pg advisory lock. + +## Global Constraints + +- Branch: `fix/backend-hardening` (already created). +- CHANGELOG entries in `docs/CHANGELOG.md`: English line then Chinese line on consecutive lines, different entries separated by blank line. +- `docs/README.md` and `docs/README_zh.md` stay content-equivalent; update both in same change. +- API contract: zero changes except B6/B7/B8 error semantics (recorded in changelog). +- Migration: self-hosted embedded SQL + `schema_migrations` + pg advisory lock. No down migrations. +- Refactor: small consumer-side interfaces in `internal/ports`, hand-written fakes in `internal/ports/portsfake`, no DI framework, no mock generators. +- CI: standard GitHub Actions syntax in `.github/workflows/`; Gitea Actions compatible. Runner-less until Gitea runner registered; local gate mandatory before each batch merge. +- Local gate before each batch merge: dev compose PG+Redis up, then `go vet ./... && gofmt -l . && go test -p 1 -count=1 ./...` confirming 0 skip, plus `scripts/smoke.sh`. + +--- + +## Part A — Migration System + CI + Quick Fixes + +### Task 1: Migration System + +**Files:** +- Create: `backend/internal/db/migrations/0001_baseline.sql` +- Modify: `backend/internal/db/db.go` +- Delete: `backend/internal/db/schema.sql` (after copying to migrations) + +**Interfaces:** +- Consumes: `pgxpool.Pool` from `db.Connect` +- Produces: `db.Migrate(ctx, pool) error` — same signature, different internals; `schema_migrations` table auto-created + +- [ ] **Step 1: Create baseline migration file** + +Copy current `backend/internal/db/schema.sql` content verbatim into `backend/internal/db/migrations/0001_baseline.sql`. Content is all `CREATE TABLE IF NOT EXISTS` and `CREATE INDEX IF NOT EXISTS` statements for users, libraries, books, reading_progress, bookmarks. + +- [ ] **Step 2: Rewrite `db.go` with migration system** + +Replace `backend/internal/db/db.go` entirely with the new migration system: + +```go +package db + +import ( + "context" + "embed" + "fmt" + "io/fs" + "log" + "regexp" + "sort" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations +var migrationsFS embed.FS + +const advisoryLockKey int64 = 0x424C4D49 // "BLMI" + +var migrationNameRe = regexp.MustCompile(`^\d{4}_[a-z0-9_]+\.sql$`) + +func Connect(ctx context.Context, url string) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, err + } + cfg.MaxConns = 10 + return pgxpool.NewWithConfig(ctx, cfg) +} + +func Migrate(ctx context.Context, p *pgxpool.Pool) error { + // 1. Acquire advisory lock — serializes concurrent replicas. + if _, err := p.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockKey); err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + defer func() { + if _, err := p.Exec(ctx, "SELECT pg_advisory_unlock($1)", advisoryLockKey); err != nil { + log.Printf("advisory unlock: %v", err) + } + }() + + // 2. Create tracking table. + if _, err := p.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now())`); err != nil { + return fmt.Errorf("create schema_migrations: %w", err) + } + + // 3. Read + validate embedded files. + entries, err := fs.ReadDir(migrationsFS, "migrations") + if err != nil { + return fmt.Errorf("read migrations dir: %w", err) + } + var files []string + for _, e := range entries { + name := e.Name() + if !migrationNameRe.MatchString(name) { + panic(fmt.Sprintf("invalid migration filename: %q", name)) + } + files = append(files, name) + } + sort.Strings(files) + + // 4. Baseline: empty schema_migrations + books exists → mark 0001 applied. + var count int + if err := p.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&count); err != nil { + return fmt.Errorf("count migrations: %w", err) + } + if count == 0 { + var hasBooks bool + if err := p.QueryRow(ctx, "SELECT to_regclass('books') IS NOT NULL").Scan(&hasBooks); err != nil { + return fmt.Errorf("check books: %w", err) + } + if hasBooks && len(files) > 0 && strings.HasPrefix(files[0], "0001_") { + if _, err := p.Exec(ctx, + "INSERT INTO schema_migrations (version, name) VALUES ($1,$2)", + 1, files[0]); err != nil { + return fmt.Errorf("baseline: %w", err) + } + log.Printf("migration baseline: %s (existing db)", files[0]) + files = files[1:] + } + } + + // 5. Load applied set. + applied := map[int64]bool{} + rows, err := p.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return fmt.Errorf("list applied: %w", err) + } + defer rows.Close() + for rows.Next() { + var v int64 + if err := rows.Scan(&v); err != nil { + return fmt.Errorf("scan: %w", err) + } + applied[v] = true + } + if err := rows.Err(); err != nil { + return err + } + + // 6. Apply pending, each in own transaction. + for _, name := range files { + v := parseVersion(name) + if applied[v] { + continue + } + sql, err := fs.ReadFile(migrationsFS, "migrations/"+name) + if err != nil { + return fmt.Errorf("read %s: %w", name, err) + } + tx, err := p.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(sql)); err != nil { + tx.Rollback(ctx) + return fmt.Errorf("exec %s: %w", name, err) + } + if _, err := tx.Exec(ctx, + "INSERT INTO schema_migrations (version, name) VALUES ($1,$2)", v, name); err != nil { + tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + log.Printf("migration applied: %s", name) + } + return nil +} + +func parseVersion(name string) int64 { + parts := strings.SplitN(name, "_", 2) + var v int64 + fmt.Sscanf(parts[0], "%d", &v) + return v +} +``` + +- [ ] **Step 3: Delete old schema.sql** + +```bash +git rm backend/internal/db/schema.sql +``` + +- [ ] **Step 4: Verify build + tests** + +```bash +cd backend && go build ./... +DATABASE_URL=postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable \ + go test -p 1 -count=1 ./... +``` + +- [ ] **Step 5: Verify baseline on existing DB** + +On a DB with old schema (no `schema_migrations`): +1. Start app → verify `SELECT * FROM schema_migrations` has one row (version=1). +2. Verify existing data untouched (`SELECT count(*) FROM books` same as before). +3. Verify no DDL re-executed (no errors in log). + +- [ ] **Step 6: Commit** + +```bash +git add backend/internal/db/migrations/0001_baseline.sql backend/internal/db/db.go +git rm backend/internal/db/schema.sql +git commit -m "feat(db): ordered migration system with advisory lock and baseline detection" +``` + +--- + +### Task 2: CI Workflow + +**Files:** +- Create: `.github/workflows/ci.yml` + +- [ ] **Step 1: Create workflow** + +```yaml +name: CI +on: + push: + branches: [master, 'fix/**', 'feat/**'] + pull_request: + branches: [master] +jobs: + backend: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: {POSTGRES_USER: booklib, POSTGRES_PASSWORD: booklib, POSTGRES_DB: booklib} + ports: ['5432:5432'] + options: --health-cmd pg_isready --health-interval 5s --health-timeout 3s --health-retries 10 + redis: + image: redis:7 + ports: ['6379:6379'] + options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: {go-version-file: backend/go.mod} + - run: | + cd backend && OUT=$(gofmt -l .) && [ -z "$OUT" ] || { echo "gofmt:"; echo "$OUT"; exit 1; } + - run: cd backend && go vet ./... + - run: cd backend && go test -p 1 -count=1 ./... + env: + DATABASE_URL: postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable + REDIS_URL: redis://localhost:6379/0 + frontend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: {node-version: 20} + - run: cd frontend && npm ci && npm run check +``` + +- [ ] **Step 2: Validate (optional)** + +```bash +go run github.com/rhysd/actionlint/cmd/actionlint@latest .github/workflows/ci.yml +``` + +- [ ] **Step 3: Commit** + +```bash +git add .github/workflows/ci.yml +git commit -m "ci: add GitHub Actions workflow (Gitea Actions compatible)" +``` + +--- + +### Task 3: Fix B14 — log.Fatalf bypasses shutdown + +**Files:** +- Modify: `backend/cmd/webui/main.go:43-55` + +- [ ] **Step 1: Replace serve goroutine + wait logic** + +Replace the block from `srv :=` through `<-ctx.Done()` in `main.go`: + +```go + serveErr := make(chan error, 1) + srv := &http.Server{Addr: cfg.Addr, Handler: api.NewRouter(cfg, st, rdb, sc), + ReadHeaderTimeout: 10 * time.Second} + go func() { + log.Printf("listening on %s", cfg.Addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + serveErr <- err + } + close(serveErr) + }() + select { + case <-ctx.Done(): + case err := <-serveErr: + if err != nil { + log.Printf("serve: %v", err) + } + } + stop() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("shutdown: %v", err) + } +``` + +- [ ] **Step 2: Build + smoke** + +```bash +cd backend && go build ./cmd/webui && scripts/smoke.sh +``` + +- [ ] **Step 3: Commit** + +```bash +git add backend/cmd/webui/main.go +git commit -m "fix(main): channel-based serve error instead of log.Fatalf in goroutine (B14)" +``` + +--- + +### Task 4: Fix B15 — Config validation + +**Files:** +- Modify: `backend/internal/config/config.go` +- Modify: `backend/internal/config/config_test.go` + +- [ ] **Step 1: Add failing tests to config_test.go** + +```go +func TestLoadDatabaseURLRequired(t *testing.T) { + t.Setenv("JWT_SECRET", "x") + t.Setenv("DATABASE_URL", "") + if _, err := Load(); err == nil { + t.Fatal("empty DATABASE_URL must fail") + } +} + +func TestLoadDatabaseURLMalformed(t *testing.T) { + t.Setenv("JWT_SECRET", "x") + t.Setenv("DATABASE_URL", "not a url") + if _, err := Load(); err == nil { + t.Fatal("malformed DATABASE_URL must fail") + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +```bash +cd backend && go test -run "TestLoadDatabase" -v ./internal/config/ +``` +Expected: `TestLoadDatabaseURLRequired` FAIL (empty URL currently accepted). + +- [ ] **Step 3: Add validation in config.Load** + +Before the `return &Config{...}` in `Load()`, add: + +```go + dbURL := env("DATABASE_URL", "") + if dbURL == "" { + return nil, fmt.Errorf("DATABASE_URL is required") + } + if _, perr := pgxpool.ParseConfig(dbURL); perr != nil { + return nil, fmt.Errorf("DATABASE_URL: %w", perr) + } + if env("REDIS_URL", "") == "" { + log.Printf("redis disabled: rate-limit/scan-lock/page-cache off") + } +``` + +Add imports: `"log"`, `"github.com/jackc/pgx/v5/pgxpool"`. + +- [ ] **Step 4: Run tests to verify pass** + +```bash +cd backend && go test -v ./internal/config/ +``` + +- [ ] **Step 5: Commit** + +```bash +git add backend/internal/config/ +git commit -m "fix(config): validate DATABASE_URL, log redis disabled (B15)" +``` + +--- + +### Task 5: Fix B17 — smoke.sh outdated root_path + +**Files:** +- Modify: `scripts/smoke.sh` + +- [ ] **Step 1: Remove `root_path` from library creation JSON** + +Find the `curl` POST to `/libraries` (around line 32) and change: +```bash +-d '{"name":"smoke","root_path":"/data/books/smoke-books"}' +``` +to: +```bash +-d '{"name":"smoke"}' +``` + +- [ ] **Step 2: Verify** + +```bash +scripts/smoke.sh +``` + +- [ ] **Step 3: Commit** + +```bash +git add scripts/smoke.sh +git commit -m "fix(smoke): drop ignored root_path field, align with API contract (B17)" +``` + +--- + +### Task 6: Batch A docs + +**Files:** +- Modify: `docs/CHANGELOG.md` +- Modify: `docs/README.md`, `docs/README_zh.md` + +- [ ] **Step 1: Prepend CHANGELOG entries** + +``` +## [Unreleased] + +### Changed +- Add ordered migration system with schema_migrations tracking and pg advisory lock. Existing databases auto-baselined. +- 新增有序迁移系统,通过 schema_migrations 表和 pg advisory lock 实现安全多副本 schema 演进,已有数据库自动基线化。 + +### Fixed +- Serve goroutine log.Fatalf replaced with channel-based shutdown (B14). +- 服务 goroutine 的 log.Fatalf 改为 channel 通知,确保优雅关停(B14)。 +- DATABASE_URL validated at startup; empty REDIS_URL logs clear message (B15). +- 启动时校验 DATABASE_URL;空 REDIS_URL 记录明确日志(B15)。 +- smoke.sh aligned with current API contract (B17). +- smoke.sh 对齐当前 API 契约(B17)。 + +### Added +- CI workflow (.github/workflows/ci.yml) for GitHub/Gitea Actions. +- CI 工作流,兼容 GitHub Actions 和 Gitea Actions。 +``` + +- [ ] **Step 2: Update README dual versions** + +Add "Schema Migration" section (how to add `NNNN_description.sql`, never modify applied files, no down migrations, baseline auto-upgrade). Add "CI" section (workflow location, Gitea/GitHub setup, local gate command). + +- [ ] **Step 3: Run local gate** + +```bash +cd backend && go vet ./... && gofmt -l . && \ +DATABASE_URL=postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable \ +REDIS_URL=redis://localhost:6379/0 \ +go test -p 1 -count=1 ./... +``` +Confirm 0 skip. Then run `scripts/smoke.sh`. + +- [ ] **Step 4: Commit** + +```bash +git add docs/ +git commit -m "docs: changelog + README for batch A (migrations, CI, B14/B15/B17)" +``` + +--- + +## Part B — Bug Fixes B1–B13 + +Each bug: **write failing test → fix → verify → continue**. Tests use real PG/Redis (integration) until Part C converts applicable ones to fake-based unit tests. + +### Task 7: B1 — Rate limiter permanent lockout + +**Files:** +- Modify: `backend/internal/redispkg/redis.go` (IncrWindow) +- Modify: `backend/internal/redispkg/redis_test.go` + +- [ ] **Step 1: Add test** + +```go +func TestIncrWindowSetsTTL(t *testing.T) { + url := os.Getenv("REDIS_URL") + if url == "" { t.Skip("REDIS_URL not set") } + r := New(url) + ctx := context.Background() + key := "test:ttl:" + t.Name() + r.c.Del(ctx, key) + n := r.IncrWindow(ctx, key, 5*time.Second) + if n != 1 { t.Fatalf("first = %d", n) } + ttl, err := r.c.TTL(ctx, key).Result() + if err != nil { t.Fatal(err) } + if ttl <= 0 { t.Fatalf("TTL must be positive, got %v", ttl) } +} +``` + +- [ ] **Step 2: Fix IncrWindow with Lua script** + +```go +var incrWindowScript = redis.NewScript(` +local n = redis.call('INCR', KEYS[1]) +if n == 1 then redis.call('EXPIRE', KEYS[1], ARGV[1]) end +return n`) + +func (r *R) IncrWindow(ctx context.Context, key string, ttl time.Duration) int { + if r.c == nil { return 1 } + n, err := incrWindowScript.Run(ctx, r.c, []string{key}, int(ttl.Seconds())).Int() + if err != nil { return 1 } + return n +} +``` + +- [ ] **Step 3: Run + commit** + +```bash +REDIS_URL=redis://localhost:6379/0 go test -v ./internal/redispkg/ +git add backend/internal/redispkg/ && git commit -m "fix(redis): atomic INCR+EXPIRE via Lua (B1)" +``` + +--- + +### Task 8: B2+B3 — Lock rand error + unlock context + +**Files:** +- Modify: `backend/internal/redispkg/redis.go` (Lock method) + +- [ ] **Step 1: Fix token generation (B2)** + +In `Lock`, replace `rand.Read(b)` / `tok := hex.EncodeToString(b)` with: + +```go + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + log.Printf("rand.Read: %v (degrading to no-lock)", err) + return noop, true + } + tok := hex.EncodeToString(b) +``` + +- [ ] **Step 2: Fix unlock context (B3)** + +Replace the returned unlock closure's Eval call: + +```go + return func() { + if err := r.c.Eval(context.WithoutCancel(ctx), + "if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", + []string{key}, tok).Err(); err != nil { + log.Printf("unlock %s: %v", key, err) + } + }, true +``` + +- [ ] **Step 3: Run + commit** + +```bash +go test -v ./internal/redispkg/ +git add backend/internal/redispkg/ && git commit -m "fix(redis): handle rand error + unlock survives ctx cancel (B2, B3)" +``` + +--- + +### Task 9: B4 — Truncated part reported as received + +**Files:** +- Modify: `backend/cmd/webui/handlers/uploads.go` (UploadPart method) + +- [ ] **Step 1: Fix part write to use tmp+rename** + +In `UploadPart`, replace the file-writing block (from `p := filepath.Join(dir, "parts", ...)` through the error handling): + +```go + p := filepath.Join(dir, "parts", strconv.FormatInt(idx, 10)) + tmp := p + ".tmp" + f, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if e != nil { + err(c, http.StatusInternalServerError, "internal", "create part") + return + } + n, e := io.Copy(f, c.Request.Body) + f.Close() + if e != nil || n != hi-lo { + os.Remove(tmp) + var mbe *http.MaxBytesError + code, msg := "too_large", "part size mismatch" + if errors.As(e, &mbe) { msg = "part exceeds declared size" } + err(c, http.StatusRequestEntityTooLarge, code, msg) + return + } + if e := os.Rename(tmp, p); e != nil { + os.Remove(tmp) + err(c, http.StatusInternalServerError, "internal", "rename part") + return + } + c.JSON(http.StatusAccepted, gin.H{"accepted": true}) +``` + +- [ ] **Step 2: Run + commit** + +```bash +go test -v ./cmd/webui/handlers/ -run Upload +git add backend/cmd/webui/handlers/uploads.go && git commit -m "fix(upload): write part to .tmp then rename (B4)" +``` + +--- + +### Task 10: B5 — DeleteUser last-admin TOCTOU + +**Files:** +- Modify: `backend/internal/store/store.go` +- Modify: `backend/cmd/webui/handlers/users.go` + +- [ ] **Step 1: Add test to store_test.go** + +```go +func TestDeleteUserLastAdmin(t *testing.T) { + s := setup(t) + ctx := context.Background() + id, _ := s.CreateUser(ctx, "onlyadmin", "h", "admin") + err := s.DeleteUser(ctx, id) + if !errors.Is(err, ErrLastAdmin) { + t.Fatalf("want ErrLastAdmin, got %v", err) + } + if _, err := s.GetUserByID(ctx, id); err != nil { + t.Fatal("admin should still exist") + } +} + +func TestDeleteUserNonLastAdmin(t *testing.T) { + s := setup(t) + ctx := context.Background() + id1, _ := s.CreateUser(ctx, "a1", "h", "admin") + s.CreateUser(ctx, "a2", "h", "admin") + if err := s.DeleteUser(ctx, id1); err != nil { + t.Fatalf("non-last admin delete: %v", err) + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +Expected: `ErrLastAdmin` undefined. + +- [ ] **Step 3: Add ErrLastAdmin + transactional DeleteUser to store.go** + +Add sentinel: +```go +var ErrLastAdmin = errors.New("cannot delete the last admin") +``` + +Replace `DeleteUser`: +```go +func (s *Store) DeleteUser(ctx context.Context, id int64) error { + tx, err := s.P.Begin(ctx) + if err != nil { return err } + defer tx.Rollback(ctx) + var role string + if err := tx.QueryRow(ctx, "SELECT role FROM users WHERE id=$1 FOR UPDATE", id).Scan(&role); err != nil { + return err + } + if role == "admin" { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM users WHERE role='admin'").Scan(&n); err != nil { + return err + } + if n <= 1 { return ErrLastAdmin } + } + if _, err := tx.Exec(ctx, "DELETE FROM users WHERE id=$1", id); err != nil { return err } + return tx.Commit(ctx) +} +``` + +Remove the old exported `CountAdmins` method. + +- [ ] **Step 4: Update users.go handler** + +Replace `DeleteUser` handler: + +```go +func (h *H) DeleteUser(c *gin.Context) { + id, e := strconv.ParseInt(c.Param("id"), 10, 64) + if e != nil { err(c, 400, "bad_request", "bad id"); return } + if id == uid(c) { err(c, 400, "bad_request", "cannot delete yourself"); return } + if e := h.st.DeleteUser(c, id); e != nil { + if errors.Is(e, pgx.ErrNoRows) { err(c, 404, "not_found", "no such user"); return } + if errors.Is(e, store.ErrLastAdmin) { err(c, 400, "bad_request", "cannot delete the last admin"); return } + dbErr(c, e); return + } + c.Status(204) +} +``` + +Remove the old `CountAdmins` + `GetUserByID` + role check block. Add `"booklib/internal/store"` import. + +- [ ] **Step 5: Run + commit** + +```bash +DATABASE_URL=... go test -v ./internal/store/ ./cmd/webui/handlers/ +git add backend/internal/store/store.go backend/cmd/webui/handlers/users.go +git commit -m "fix(store): transactional last-admin check in DeleteUser (B5)" +``` + +--- + +### Task 11: B6+B7 — Error semantics fixes + +**Files:** +- Modify: `backend/cmd/webui/handlers/libraries.go` (Upload method, B6) +- Modify: `backend/cmd/webui/handlers/auth.go` (Me method, B7) + +- [ ] **Step 1: Fix B6 — Upload io.Copy error mapping** + +In `libraries.go` `Upload` method, replace the `io.Copy` error block: + +```go + if _, e := io.Copy(out, src); e != nil { + out.Close() + os.Remove(tmp) + var mbe *http.MaxBytesError + if errors.As(e, &mbe) { + err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit") + return + } + err(c, http.StatusInternalServerError, "internal", "upload failed") + return + } +``` + +- [ ] **Step 2: Fix B7 — Me handler error mapping** + +In `auth.go` `Me` method, replace: + +```go +func (h *H) Me(c *gin.Context) { + u, qerr := h.st.GetUserByID(c, uid(c)) + if qerr != nil { + if errors.Is(qerr, pgx.ErrNoRows) { + err(c, http.StatusUnauthorized, "unauthorized", "no such user") + return + } + dbErr(c, qerr) + return + } + c.JSON(http.StatusOK, gin.H{"id": u.ID, "username": u.Username, "role": u.Role}) +} +``` + +- [ ] **Step 3: Run + commit** + +```bash +go test -v ./cmd/webui/handlers/ +git add backend/cmd/webui/handlers/libraries.go backend/cmd/webui/handlers/auth.go +git commit -m "fix(handlers): correct error mapping for Upload (B6) and Me (B7)" +``` + +--- + +### Task 12: B8 — Reserved library names + +**Files:** +- Create: `backend/internal/media/reserved.go` +- Create: `backend/internal/media/reserved_test.go` +- Modify: `backend/cmd/webui/handlers/libraries.go` + +- [ ] **Step 1: Create reserved.go** + +```go +package media + +import "strings" + +var reservedNames = map[string]bool{ + "cache": true, ".uploads": true, ".trash": true, +} + +func IsReservedName(name string) bool { + return reservedNames[strings.ToLower(name)] +} +``` + +- [ ] **Step 2: Create reserved_test.go** + +```go +package media + +import "testing" + +func TestIsReservedName(t *testing.T) { + for _, n := range []string{"cache", "Cache", "CACHE", ".uploads", ".Uploads", ".trash"} { + if !IsReservedName(n) { t.Errorf("%q should be reserved", n) } + } + for _, n := range []string{"comics", "books", "my-library"} { + if IsReservedName(n) { t.Errorf("%q should not be reserved", n) } + } +} +``` + +- [ ] **Step 3: Add check in CreateLibrary handler** + +In `libraries.go` `CreateLibrary`, after `SafeName` check, add: + +```go + if media.IsReservedName(safe) { + err(c, http.StatusBadRequest, "bad_request", "reserved_name") + return + } +``` + +Import `"booklib/internal/media"`. + +- [ ] **Step 4: Run + commit** + +```bash +go test -v ./internal/media/ ./cmd/webui/handlers/ -run "Reserved\|CreateLibrary" +git add backend/internal/media/ backend/cmd/webui/handlers/libraries.go +git commit -m "fix(library): reject reserved names with 400 reserved_name (B8)" +``` + +--- + +### Task 13: B9-① — Scan lock renewal + +**Files:** +- Modify: `backend/internal/redispkg/redis.go` +- Modify: `backend/internal/redispkg/redis_test.go` +- Modify: `backend/internal/scanner/scanner.go` + +- [ ] **Step 1: Add test** + +```go +func TestScanLockRenewal(t *testing.T) { + url := os.Getenv("REDIS_URL") + if url == "" { t.Skip("REDIS_URL not set") } + r := New(url) + ctx, key := context.Background(), "test:scanlock:"+t.Name() + unlock, ok := r.ScanLock(ctx, key, 2*time.Second) + if !ok { t.Fatal("should acquire") } + time.Sleep(3 * time.Second) // would expire without renewal + _, ok2 := r.ScanLock(ctx, key, 2*time.Second) + if ok2 { t.Fatal("should still be held") } + unlock() +} +``` + +- [ ] **Step 2: Add ScanLock to redis.go** + +```go +func (r *R) ScanLock(ctx context.Context, key string, ttl time.Duration) (func(), bool) { + noop := func() {} + if r.c == nil { return noop, true } + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + log.Printf("rand: %v", err); return noop, true + } + tok := hex.EncodeToString(b) + ok, err := r.c.SetNX(ctx, key, tok, ttl).Result() + if err != nil { + log.Printf("scanlock %s: %v (proceeding)", key, err) + return noop, true + } + if !ok { return noop, false } + done := make(chan struct{}) + go func() { + tk := time.NewTicker(ttl / 2); defer tk.Stop() + for { + select { + case <-done: return + case <-tk.C: + r.c.Eval(context.Background(), + `if redis.call('get',KEYS[1])==ARGV[1] then + return redis.call('expire',KEYS[1],ARGV[2]) end`, + []string{key}, tok, int(ttl.Seconds())) + } + } + }() + return func() { + close(done) + r.c.Eval(context.WithoutCancel(ctx), + "if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end", + []string{key}, tok) + }, true +} +``` + +- [ ] **Step 3: Update scanner to use ScanLock** + +In `scanner.go` `ScanLibrary`, change: +```go + unlock, ok := s.rdb.Lock(ctx, fmt.Sprintf("scan:%d", lib.ID), 5*time.Minute) +``` +to: +```go + unlock, ok := s.rdb.ScanLock(ctx, fmt.Sprintf("scan:%d", lib.ID), 5*time.Minute) +``` + +- [ ] **Step 4: Run + commit** + +```bash +REDIS_URL=redis://localhost:6379/0 go test -v -timeout 10s ./internal/redispkg/ +go test -v ./internal/scanner/ +git add backend/internal/redispkg/ backend/internal/scanner/scanner.go +git commit -m "fix(redis): ScanLock with auto-renewal for scanner (B9-①)" +``` + +--- + +### Task 14: B10+B11 — Silent error logging fixes + +**Files:** +- Modify: `backend/internal/scanner/scanner.go` (cover, add, update) +- Modify: `backend/cmd/webui/handlers/content.go` (ServeCover) + +- [ ] **Step 1: Fix scanner add/update SetBookState (B10)** + +In both `add` and `update`, change: +```go + s.st.SetBookState(ctx, id, "error", idxErr.Error()) +``` +to: +```go + if e := s.st.SetBookState(ctx, id, "error", idxErr.Error()); e != nil { + log.Printf("scan: SetBookState %s: %v", rel, e) + } +``` + +- [ ] **Step 2: Fix scanner cover write (B11)** + +Replace the cover write section: +```go + tmp := filepath.Join(dir, "cover"+ext+".tmp") + dst := filepath.Join(dir, "cover"+ext) + if e := os.WriteFile(tmp, img, 0o644); e != nil { + log.Printf("scan: cover tmp %s: %v", rel, e) + os.Remove(tmp) + return + } + if e := os.Rename(tmp, dst); e != nil { + log.Printf("scan: cover rename %s: %v", rel, e) + os.Remove(tmp) + return + } +``` + +- [ ] **Step 3: Fix ServeCover self-heal (B11)** + +In `content.go`, replace the self-heal write block: +```go + tmp := fmt.Sprintf("%s.tmp-%d", dst, time.Now().UnixNano()) + if e := os.WriteFile(tmp, img, 0o644); e != nil { + log.Printf("serve: cover write: %v", e) + os.Remove(tmp) + } else if e := os.Rename(tmp, dst); e != nil { + log.Printf("serve: cover rename: %v", e) + os.Remove(tmp) + } else { + http.ServeFile(c.Writer, c.Request, dst) + } +``` + +Add `"log"` import if missing. + +- [ ] **Step 4: Run + commit** + +```bash +go test -v ./internal/scanner/ ./cmd/webui/handlers/ +git add backend/internal/scanner/scanner.go backend/cmd/webui/handlers/content.go +git commit -m "fix: check cover write errors + log SetBookState failures (B10, B11)" +``` + +--- + +### Task 15: B12 — uniquePath race + +**Files:** +- Modify: `backend/cmd/webui/handlers/libraries.go` (Upload method) + +- [ ] **Step 1: Add retry loop in Upload handler** + +In `Upload`, replace the single `uniquePath` + `OpenFile` block with a retry: + +```go + var dst string + var out *os.File + for attempt := 0; attempt < 5; attempt++ { + var e error + dst, e = h.uniquePath(root, name) + if e != nil { err(c, http.StatusForbidden, "forbidden", e.Error()); return } + tmp := dst + ".upload-" + strconv.FormatInt(time.Now().UnixNano(), 36) + out, e = os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if e == nil { break } + if !os.IsExist(e) { err(c, 500, "internal", "create tmp"); return } + } + if out == nil { + err(c, http.StatusConflict, "conflict", "concurrent upload conflict") + return + } +``` + +- [ ] **Step 2: Run + commit** + +```bash +go test -v ./cmd/webui/handlers/ -run Upload +git add backend/cmd/webui/handlers/libraries.go +git commit -m "fix(upload): retry on O_EXCL collision (B12)" +``` + +--- + +### Task 16: B13 — RowsAffected before err + +**Files:** +- Modify: `backend/internal/store/store.go` (UpdateBookmarkNote, DeleteBookmark) + +- [ ] **Step 1: Fix both methods** + +```go +func (s *Store) UpdateBookmarkNote(ctx context.Context, userID, id int64, note string) (bool, error) { + res, err := s.P.Exec(ctx, `UPDATE bookmarks SET note=$3 WHERE id=$1 AND user_id=$2`, id, userID, note) + if err != nil { return false, err } + return res.RowsAffected() > 0, nil +} + +func (s *Store) DeleteBookmark(ctx context.Context, userID, id int64) (bool, error) { + res, err := s.P.Exec(ctx, `DELETE FROM bookmarks WHERE id=$1 AND user_id=$2`, id, userID) + if err != nil { return false, err } + return res.RowsAffected() > 0, nil +} +``` + +- [ ] **Step 2: Run + commit** + +```bash +go test -v ./internal/store/ +git add backend/internal/store/store.go +git commit -m "fix(store): check err before RowsAffected (B13)" +``` + +--- + +### Task 17: B9-② — Scanner per-library single-flight + +**Files:** +- Modify: `backend/internal/scanner/scanner.go` + +- [ ] **Step 1: Add sync.Map + scanOnce wrapper** + +Add field to Scanner: +```go + flights sync.Map // map[int64]*sync.WaitGroup +``` + +Add method: +```go +func (s *Scanner) scanOnce(ctx context.Context, lib store.Library) { + wg := &sync.WaitGroup{} + wg.Add(1) + if existing, loaded := s.flights.LoadOrStore(lib.ID, wg); loaded { + existing.(*sync.WaitGroup).Wait() + return + } + defer func() { s.flights.Delete(lib.ID); wg.Done() }() + s.ScanLibrary(ctx, lib) +} +``` + +Update `Run` to call `s.scanOnce(ctx, l)` and `ScanLibraryByID` to call `s.scanOnce(ctx, lib)`. + +- [ ] **Step 2: Run + commit** + +```bash +go test -v ./internal/scanner/ +git add backend/internal/scanner/scanner.go +git commit -m "fix(scanner): per-library single-flight (B9-②)" +``` + +--- + +### Task 18: Batch B docs + local gate + +**Files:** +- Modify: `docs/CHANGELOG.md` + +- [ ] **Step 1: Prepend CHANGELOG entries for B1–B13** + +Add entries for each fix in the `[Unreleased]` section. Format per global constraints (English+Chinese, blank between entries). See spec §S2 for the full list. + +- [ ] **Step 2: Run full local gate** + +```bash +cd backend && go vet ./... && gofmt -l . +DATABASE_URL=postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable \ +REDIS_URL=redis://localhost:6379/0 \ +go test -p 1 -count=1 ./... +scripts/smoke.sh +``` +Confirm 0 skip. + +- [ ] **Step 3: Commit** + +```bash +git add docs/CHANGELOG.md +git commit -m "docs: changelog for batch B (B1-B13)" +``` + +--- + +## Part C — Structure Restructure + Interfaces + +### Task 19: Define ports package + +**Files:** +- Create: `backend/internal/ports/ports.go` +- Create: `backend/internal/ports/errors.go` + +- [ ] **Step 1: Create ports.go** + +Define consumer-side interfaces. Value types use `store.*` (ports imports store for types only — one-way dependency): + +```go +package ports + +import ( + "context" + "time" + "booklib/internal/store" +) + +type UserStore interface { + CountUsers(ctx context.Context) (int, error) + CreateUser(ctx context.Context, username, hash, role string) (int64, error) + GetUserByName(ctx context.Context, username string) (store.User, error) + GetUserByID(ctx context.Context, id int64) (store.User, error) + ListUsers(ctx context.Context) ([]store.User, error) + DeleteUser(ctx context.Context, id int64) error +} + +type LibraryStore interface { + CreateLibrary(ctx context.Context, name, root string) (int64, error) + ListLibraries(ctx context.Context) ([]store.Library, error) + GetLibrary(ctx context.Context, id int64) (store.Library, error) +} + +type BookStore interface { + InsertBook(ctx context.Context, libID int64, path, title, format string, size, modTS int64, pageCount int) (int64, error) + GetBook(ctx context.Context, id int64) (store.Book, error) + ListBookMeta(ctx context.Context, libID int64) (map[string]store.BookMeta, error) + UpdateBookFile(ctx context.Context, id, size, modTS int64, pageCount int) error + DeleteBookByPath(ctx context.Context, libID int64, path string) error + DeleteBook(ctx context.Context, id int64) error + SetBookState(ctx context.Context, id int64, state, msg string) error + ListBooks(ctx context.Context, libID int64, q, prefix string, userID int64) ([]store.BookView, error) + BookHashes(ctx context.Context) (map[int64][2]int64, error) +} + +type ProgressStore interface { + UpsertProgress(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64) error + GetProgress(ctx context.Context, userID, libID int64, bookPath string) (store.Progress, error) + ListProgress(ctx context.Context, userID int64) ([]store.Progress, error) +} + +type BookmarkStore interface { + InsertBookmark(ctx context.Context, userID, libID int64, bookPath string, locator []byte, percent float64, note string) (int64, error) + ListBookmarks(ctx context.Context, userID, libID int64, bookPath string) ([]store.Bookmark, error) + UpdateBookmarkNote(ctx context.Context, userID, id int64, note string) (bool, error) + DeleteBookmark(ctx context.Context, userID, id int64) (bool, error) +} + +type PageCache interface { + Get(ctx context.Context, key string) (string, bool) + Set(ctx context.Context, key, val string, ttl time.Duration) +} + +type RateLimiter interface { + IncrWindow(ctx context.Context, key string, ttl time.Duration) int +} + +type ScanLocker interface { + ScanLock(ctx context.Context, key string, ttl time.Duration) (func(), bool) +} + +type Scanner interface { + ScanLibraryByID(ctx context.Context, id int64) +} +``` + +Note: `UploadSessions` and `Media` interfaces will be defined after `internal/upload` and `internal/media` are created (Tasks 22-23). + +- [ ] **Step 2: Create errors.go** + +```go +package ports + +import "booklib/internal/store" + +// Re-export sentinel errors so handlers use errors.Is via ports. +var ErrLastAdmin = store.ErrLastAdmin + +// IsUniqueViolation consolidates the 3 copies of pg 23505 check. +func IsUniqueViolation(err error) bool { return store.IsUniqueViolation(err) } +``` + +- [ ] **Step 3: Add IsUniqueViolation + ErrUniqueViolation to store** + +In `store.go`, add: +```go +var ErrUniqueViolation = errors.New("unique violation") + +func IsUniqueViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23505" +} +``` + +Update `ports/errors.go` to also re-export: +```go +var ErrUniqueViolation = store.ErrUniqueViolation +``` + +- [ ] **Step 4: Verify build** + +```bash +cd backend && go build ./... +``` + +- [ ] **Step 5: Commit** + +```bash +git add backend/internal/ports/ backend/internal/store/store.go +git commit -m "feat(ports): define consumer-side interfaces and shared errors" +``` + +--- + +### Task 20: Split store package + +**Files:** +- Create: `backend/internal/store/users.go` +- Create: `backend/internal/store/libraries.go` +- Create: `backend/internal/store/books.go` +- Create: `backend/internal/store/progress.go` +- Create: `backend/internal/store/bookmarks.go` +- Modify: `backend/internal/store/store.go` (keep types + ctor + shared helpers only) +- Modify: `backend/internal/store/store_test.go` (extract per-file tests) + +- [ ] **Step 1: Extract methods by aggregate** + +Move methods from `store.go` into per-aggregate files: +- `users.go`: `CountUsers`, `CreateUser`, `GetUserByName`, `GetUserByID`, `ListUsers`, `DeleteUser`, `scanUser`, `CountAdmins` (make unexported: `countAdmins`) +- `libraries.go`: `CreateLibrary`, `ListLibraries`, `GetLibrary` +- `books.go`: `InsertBook`, `GetBook`, `ListBookMeta`, `UpdateBookFile`, `DeleteBookByPath`, `DeleteBook`, `SetBookState`, `ListBooks`, `BookHashes`. Delete `ListBookIDs` (dead code per spec). +- `progress.go`: `UpsertProgress`, `GetProgress`, `ListProgress` +- `bookmarks.go`: `InsertBookmark`, `ListBookmarks`, `UpdateBookmarkNote`, `DeleteBookmark` + +Each file: `package store`, receives methods on `*Store`. + +- [ ] **Step 2: Slim down store.go** + +Keep: type definitions (`Store`, `User`, `Library`, `Book`, `BookMeta`, `BookView`, `Progress`, `Bookmark`), `New()`, `IsUniqueViolation()`, `ErrLastAdmin`, column constants (`userCols`, `bookCols`). + +Make `P` field unexported (`p *pgxpool.Pool`). Update all method files to use `s.p` instead of `s.P`. Update `store_test.go` setup if it uses `s.P`. + +- [ ] **Step 3: Verify build + tests** + +```bash +cd backend && go build ./... && DATABASE_URL=... go test ./internal/store/ +``` + +- [ ] **Step 4: Commit** + +```bash +git add backend/internal/store/ +git commit -m "refactor(store): split into per-aggregate files, unexport pool" +``` + +--- + +### Task 21: Add bookfile utilities + +**Files:** +- Modify: `backend/internal/bookfile/bookfile.go` (add Contains) +- Create: `backend/internal/bookfile/reader.go` (OpenReaderAt) + +- [ ] **Step 1: Add Contains function** + +```go +// Contains reports whether child is inside parent (EvalSymlinks semantics). +func Contains(parent, child string) bool { + p, err := filepath.EvalSymlinks(parent) + if err != nil { p = filepath.Clean(parent) } + c, err := filepath.EvalSymlinks(child) + if err != nil { c = filepath.Clean(child) } + return c == p || strings.HasPrefix(c, p+string(os.PathSeparator)) +} +``` + +- [ ] **Step 2: Add OpenReaderAt** + +Consolidate the 3 places that open a book file + stat + get ReaderAt: + +```go +// OpenReaderAt opens a book file and returns a ReaderAt + size. +func OpenReaderAt(root, rel string) (*os.File, int64, error) { + abs := filepath.Join(root, filepath.FromSlash(rel)) + f, err := os.Open(abs) + if err != nil { return nil, 0, err } + st, err := f.Stat() + if err != nil { f.Close(); return nil, 0, err } + return f, st.Size(), nil +} +``` + +- [ ] **Step 3: Add tests** + +```go +func TestContains(t *testing.T) { + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + os.MkdirAll(sub, 0o755) + if !Contains(dir, sub) { t.Fatal("sub should be inside dir") } + if Contains(sub, dir) { t.Fatal("dir should not be inside sub") } +} +``` + +- [ ] **Step 4: Run + commit** + +```bash +go test -v ./internal/bookfile/ +git add backend/internal/bookfile/ +git commit -m "feat(bookfile): add Contains and OpenReaderAt utilities" +``` + +--- + +### Task 22: Create internal/media package + +**Files:** +- Modify: `backend/internal/media/reserved.go` (already exists from Task 12) +- Create: `backend/internal/media/media.go` (Media struct + interface methods) +- Create: `backend/internal/media/write.go` (WriteAtomic) +- Create: `backend/internal/media/chapters.go` (ChaptersOf) +- Create: `backend/internal/media/ports.go` (Media interface definition) + +- [ ] **Step 1: Create WriteAtomic** + +```go +package media + +import ( + "log" + "os" + "path/filepath" + "fmt" + "time" +) + +// WriteAtomic writes data to dst via tmp+rename. Cleans tmp only on failure. +func WriteAtomic(dir, name string, data []byte) error { + if err := os.MkdirAll(dir, 0o755); err != nil { return err } + tmp := filepath.Join(dir, fmt.Sprintf("%s.tmp-%d", name, time.Now().UnixNano())) + if err := os.WriteFile(tmp, data, 0o644); err != nil { + os.Remove(tmp) + return err + } + dst := filepath.Join(dir, name) + if err := os.Rename(tmp, dst); err != nil { + os.Remove(tmp) + return err + } + return nil +} +``` + +- [ ] **Step 2: Move ChaptersOf from content.go** + +Copy `chaptersOf` and `cbzChapter` from `handlers/content.go` into `media/chapters.go` (exported as `ChaptersOf`). The handler will call `media.ChaptersOf(idx)`. + +- [ ] **Step 3: Move cache layout functions** + +Move `DirKey`, `CoverDir`, `PagesDir`, `SweepStale`, `Hash` from `bookfile/cache.go` and `bookfile/hash.go` into `media/`. Or keep them in `bookfile` and have `media` wrap them — whichever avoids a large move. Per spec, media is the "single source of truth" for cache layout. Move them to `media/cache.go`: + +```go +package media + +import "booklib/internal/bookfile" + +// Re-export cache layout from bookfile for backward compat. +// Future: move implementations here. +var DirKey = bookfile.DirKey +var CoverDir = bookfile.CoverDir +var PagesDir = bookfile.PagesDir +var SweepStale = bookfile.SweepStale +var Hash = bookfile.Hash +``` + +Actually simpler: just have Media methods call bookfile functions directly. No re-export needed. Media is the consumer; bookfile stays as the low-level utility package. + +- [ ] **Step 4: Create Media struct** + +```go +package media + +import ( + "context" + "booklib/internal/bookfile" + "booklib/internal/config" + "booklib/internal/redispkg" +) + +type M struct { + cfg *config.Config + rdb *redispkg.R +} + +func New(cfg *config.Config, rdb *redispkg.R) *M { + return &M{cfg: cfg, rdb: rdb} +} +``` + +- [ ] **Step 5: Implement Media methods** + +Implement on `*M`: +- `EnsureCover(ctx, bookID, format, size, modTS, root, rel)` — extracted from scanner.cover +- `EnsurePage(ctx, bookID, size, modTS, root, rel, n)` — extracted from content.go Page handler +- `ChaptersOf(idx)` — from step 2 +- `PageIndex(ctx, bookID, size, modTS, root, rel)` — extracted from content.go pageIndex +- `CacheBuster(size, modTS)` — returns `bookfile.Hash(size, modTS)` + +- [ ] **Step 6: Create Media interface in ports.go** + +Add to `ports/ports.go` (or `ports/media.go`): +```go +type Media interface { + EnsureCover(ctx context.Context, bookID int64, format string, size, modTS int64, root, rel string) error + EnsurePage(ctx context.Context, bookID int64, size, modTS int64, root, rel string, n int) (string, error) + ChaptersOf(idx []string) []Chapter + PageIndex(ctx context.Context, bookID int64, size, modTS int64, root, rel string) ([]string, error) + CacheBuster(size, modTS int64) string +} +``` + +- [ ] **Step 7: Build + commit** + +```bash +go build ./... +git add backend/internal/media/ backend/internal/ports/ +git commit -m "feat(media): cache layout single source of truth + WriteAtomic + chapters" +``` + +--- + +### Task 23: Create internal/upload package + B16 + +**Files:** +- Create: `backend/internal/upload/upload.go` +- Create: `backend/internal/upload/upload_test.go` +- Create: `backend/internal/upload/ports.go` (UploadSessions interface) +- Modify: `backend/internal/scanner/scanner.go` (add Sweep to ticker, B16) + +- [ ] **Step 1: Move upload domain logic** + +Extract from `handlers/uploads.go` into `internal/upload/upload.go`: +- `uploadIDFor`, `uploadDir`, `chunkRange`, `numParts`, `uploadMeta`, `validUploadID` +- `Init(libID, name, size, chunkSize)` → returns uploadID +- `Status(uploadID)` → returns received part indices +- `PutPart(uploadID, index, body)` → writes part with tmp+rename (B4 fix already applied) +- `Complete(uploadID)` → assembles + renames to library root (uses `uniquePath`) +- `Sweep()` → deletes expired sessions (B16: moved from request path to scanner ticker) +- `UniquePath(root, name)` → shared between single-file and chunked upload + +The `upload` package takes `booksDir string` and `uploadMaxMB int64` in its constructor. + +- [ ] **Step 2: Define UploadSessions interface in ports** + +Add to `ports/ports.go`: +```go +type UploadSessions interface { + Init(ctx context.Context, libID int64, name string, size, chunkSize int64) (string, error) + Status(ctx context.Context, uploadID string) ([]int64, error) + PutPart(ctx context.Context, uploadID string, index int64, body io.Reader, maxSize int64) error + Complete(ctx context.Context, uploadID string, root string) (string, error) + Sweep(ctx context.Context) error + UniquePath(root, name string) (string, error) +} +``` + +Add to `ports/errors.go`: +```go +var ( + ErrTooLarge = errors.New("file too large") + ErrIncomplete = errors.New("upload incomplete") + ErrSizeMismatch = errors.New("size mismatch") + ErrNotFound = errors.New("not found") +) +``` + +- [ ] **Step 3: Wire Sweep into scanner ticker (B16)** + +In `scanner.go` `Run` method, add sweep call each tick: +```go +case <-t.C: + h.sweepUploads() // B16: moved from request path + libs, err := s.st.ListLibraries(ctx) + ... +``` + +The scanner needs access to the upload package. Add `up *upload.U` field to Scanner. Or have main.go pass the sweep function. Simplest: scanner gets an `upload.Sweeper` interface: +```go +type Sweeper interface { Sweep(ctx context.Context) error } +``` + +- [ ] **Step 4: Build + commit** + +```bash +go build ./... +git add backend/internal/upload/ backend/internal/scanner/ backend/internal/ports/ +git commit -m "feat(upload): extract upload subsystem + move sweep to scanner ticker (B16)" +``` + +--- + +### Task 24: Refactor scanner + +**Files:** +- Modify: `backend/internal/scanner/scanner.go` + +- [ ] **Step 1: Merge add/update into ingest** + +Consolidate the duplicated logic in `add` and `update`: + +```go +func (s *Scanner) ingest(ctx context.Context, libID, bookID int64, root, rel string, ds diskStat, isNew bool) { + format := bookfile.FormatFromExt(filepath.Base(rel)) + pageCount := 0 + var idxErr error + if format == "cbz" { + idx, err := s.zipIndex(root, rel) + pageCount = len(idx) + idxErr = err + if idxErr == nil && pageCount == 0 { + idxErr = errors.New("no images in archive") + } + } + if isNew { + id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount) + if err != nil { log.Printf("scan: insert %s: %v", rel, err); return } + bookID = id + } else { + if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil { + log.Printf("scan: update %s: %v", rel, err); return + } + } + if idxErr != nil { + if e := s.st.SetBookState(ctx, bookID, "error", idxErr.Error()); e != nil { + log.Printf("scan: SetBookState %s: %v", rel, e) + } + return + } + s.cover(ctx, bookID, root, rel, format, ds) +} +``` + +- [ ] **Step 2: Use bookfile.OpenReaderAt** + +Replace `readCover` and `zipIndex` to use `bookfile.OpenReaderAt`: + +```go +func (s *Scanner) zipIndex(root, rel string) ([]string, error) { + f, size, err := bookfile.OpenReaderAt(root, rel) + if err != nil { return nil, err } + defer f.Close() + return bookfile.PageIndex(f, size) +} +``` + +- [ ] **Step 3: Use bookfile.Contains** + +Replace the `inside` function call: +```go +if !bookfile.Contains(s.cfg.BooksDir, root) { ... } +``` + +Remove the local `inside` function. + +- [ ] **Step 4: Use media.WriteAtomic in cover** + +Replace manual tmp+rename in `cover` with `media.WriteAtomic(dir, "cover"+ext, img)`. + +- [ ] **Step 5: Run + commit** + +```bash +go test -v ./internal/scanner/ +git add backend/internal/scanner/ +git commit -m "refactor(scanner): merge add/update into ingest, use shared utilities" +``` + +--- + +### Task 25: Slim down handlers + +**Files:** +- Modify: All `backend/cmd/webui/handlers/*.go` +- Modify: `backend/cmd/webui/handlers/handlers.go` + +- [ ] **Step 1: Change H struct to use port interfaces** + +```go +type H struct { + cfg *config.Config + users ports.UserStore + libs ports.LibraryStore + books ports.BookStore + progress ports.ProgressStore + bookmarks ports.BookmarkStore + cache ports.PageCache + rl ports.RateLimiter + scanner ports.Scanner + media media.M // or ports.Media once interface finalized + upload upload.U // or ports.UploadSessions +} +``` + +Update `New()` to accept all interfaces. + +- [ ] **Step 2: Update all handler methods** + +Replace `h.st.Xxx(...)` with the appropriate port field. Deduplicate: +- `getLibrary` and `getLibRow` → single `getLib(c, id)` helper +- `strconv.ParseInt(c.Param("id"), ...)` → single `idParam(c)` helper +- `isUnique` → `ports.IsUniqueViolation` +- Path contains checks → `bookfile.Contains` + +- [ ] **Step 3: Update router.go to pass new H** + +Update `api/router.go` and `handlers.New()` signature to accept all dependencies. + +- [ ] **Step 4: Build + commit** + +```bash +go build ./... +git add backend/cmd/webui/ +git commit -m "refactor(handlers): use port interfaces, deduplicate helpers" +``` + +--- + +### Task 26: Wire main.go assembly + +**Files:** +- Modify: `backend/cmd/webui/main.go` + +- [ ] **Step 1: Update main.go to assemble all dependencies** + +```go +func main() { + cfg, err := config.Load() + if err != nil { log.Fatalf("config: %v", err) } + // ... ctx, db, migrate as before ... + st := store.New(p) + seed.Admin(ctx, st, cfg.AdminUser, cfg.AdminPassword) + rdb := redispkg.New(cfg.RedisURL) + med := media.New(cfg, rdb) + up := upload.New(cfg.BooksDir, cfg.UploadMaxMB) + sc := scanner.New(st, cfg, rdb, med, up) + + // Wire router with port interfaces. + // *store.Store satisfies UserStore, LibraryStore, etc. + // *redispkg.R satisfies PageCache, RateLimiter, ScanLocker. + r := api.NewRouter(cfg, st, st, st, st, st, rdb, rdb, sc, med, up) + // ... serve as Task 3 pattern ... +} +``` + +- [ ] **Step 2: Build + smoke** + +```bash +cd backend && go build ./cmd/webui && scripts/smoke.sh +``` + +- [ ] **Step 3: Commit** + +```bash +git add backend/cmd/webui/main.go +git commit -m "refactor(main): assemble dependencies via port interfaces" +``` + +--- + +### Task 27: Create portsfake + contract tests + handler unit tests + +**Files:** +- Create: `backend/internal/ports/portsfake/fake.go` +- Create: `backend/cmd/webui/handlers/users_unit_test.go` (and similar per handler) +- Modify: `backend/cmd/webui/api/router_test.go` (contract test) + +- [ ] **Step 1: Create hand-written fakes** + +`portsfake/fake.go` implements all port interfaces with in-memory maps: + +```go +package portsfake + +type Users struct { + m map[int64]store.User + next int64 +} + +func NewUsers() *Users { return &Users{m: map[int64]store.User{}, next: 1} } + +func (u *Users) CreateUser(_ context.Context, username, hash, role string) (int64, error) { + for _, v := range u.m { + if v.Username == username { return 0, ports.ErrUniqueViolation } + } + id := u.next; u.next++ + u.m[id] = store.User{ID: id, Username: username, PasswordHash: hash, Role: role} + return id, nil +} +// ... implement all interface methods +``` + +Similarly for Libraries, Books, Progress, Bookmarks fakes. And a simple in-memory PageCache, RateLimiter, ScanLocker. + +- [ ] **Step 2: Add router contract test** + +In `router_test.go`, pin the expected route table: + +```go +func TestRouterContract(t *testing.T) { + r := NewRouter(/* fakes */) + routes := r.Routes() + expected := map[string]string{ + "GET /api/healthz": "", + "POST /api/auth/login": "", + "GET /api/auth/me": "auth", + "GET /api/users": "auth,admin", + // ... all routes with middleware markers + } + // Assert exact match — any route change requires updating this test. +} +``` + +- [ ] **Step 3: Add handler unit tests** + +Example `users_unit_test.go`: + +```go +func TestDeleteUserLastAdminReturns400(t *testing.T) { + h := newTestH() // uses portsfake + // Seed one admin + h.users.CreateUser(ctx, "admin", "h", "admin") + // Try to delete → should get 400 + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Params = gin.Params{{Key: "id", Value: "1"}} + c.Set("uid", int64(999)) // different user + h.DeleteUser(c) + if w.Code != 400 { t.Fatalf("want 400, got %d", w.Code) } +} +``` + +- [ ] **Step 4: Run all tests (unit + integration)** + +```bash +go test -p 1 -count=1 ./... +``` + +- [ ] **Step 5: Commit** + +```bash +git add backend/internal/ports/portsfake/ backend/cmd/webui/handlers/*_unit_test.go backend/cmd/webui/api/router_test.go +git commit -m "test: add portsfake, router contract tests, handler unit tests" +``` + +--- + +### Task 28: Consolidate isUnique + dedup helpers + +**Files:** +- Modify: `backend/cmd/webui/handlers/users.go` (remove local `isUnique`) +- Modify: `backend/internal/seed/seed.go` (use `ports.IsUniqueViolation`) +- Modify: All handlers that duplicate error-check patterns + +- [ ] **Step 1: Replace all `isUnique` calls** + +Search for `isUnique(` and `pgconn.PgError` with code `23505` across the codebase. Replace with `ports.IsUniqueViolation(err)` or `store.IsUniqueViolation(err)`. + +Remove the local `isUnique` function from `handlers/users.go`. +Update `seed.go` to use `store.IsUniqueViolation`. + +- [ ] **Step 2: Consolidate path validation** + +Replace the 3 different path-contains checks with `bookfile.Contains`: +- `libraries.go` `libRoot` → `bookfile.Contains(h.cfg.BooksDir, root)` +- `books.go` `hasPrefixDir` → `bookfile.Contains` +- `scanner.go` `inside` → `bookfile.Contains` (already done in Task 24) + +Remove local helper functions. + +- [ ] **Step 3: Run + commit** + +```bash +go build ./... && go test -p 1 -count=1 ./... +git add -A && git commit -m "refactor: consolidate isUnique and path validation helpers" +``` + +--- + +### Task 29: Final docs + local gate + +**Files:** +- Modify: `docs/CHANGELOG.md` +- Modify: `docs/README.md`, `docs/README_zh.md` + +- [ ] **Step 1: Add CHANGELOG for Batch C** + +``` +### Changed +- Business logic restructured: handlers now consume small port interfaces, domain logic in internal/media and internal/upload. +- 业务逻辑重构:handlers 消费小口径 port 接口,域逻辑下沉至 internal/media 和 internal/upload。 +- Upload sweep moved from request path to scanner ticker cycle (B16). +- 上传会话清理从请求路径移至扫描器定时周期(B16)。 + +### Added +- Router contract test pins expected route table; unauthorized changes fail CI. +- 路由契约测试锁定预期路由表,未授权变更将导致 CI 失败。 +- Hand-written in-memory fakes (portsfake) enable handler unit tests without PG/Redis. +- 手写内存 fake(portsfake)实现无 PG/Redis 的 handler 单测。 +``` + +- [ ] **Step 2: Update READMEs** + +Document the new package structure (ports, media, upload). Explain the interface/fake testing approach. + +- [ ] **Step 3: Full local gate** + +```bash +cd backend && go vet ./... && gofmt -l . +DATABASE_URL=postgres://booklib:booklib@localhost:5432/booklib?sslmode=disable \ +REDIS_URL=redis://localhost:6379/0 \ +go test -p 1 -count=1 -v ./... +scripts/smoke.sh +``` + +Confirm: +- 0 skip in test output +- All 47+ original integration tests pass +- New fake-based unit tests pass +- Router contract test passes +- smoke.sh passes + +- [ ] **Step 4: Commit** + +```bash +git add docs/ +git commit -m "docs: changelog + README for batch C (restructure, contract tests)" +``` + +--- + +## Verification Checklist + +After all 29 tasks: + +1. `go vet ./...` — clean +2. `gofmt -l .` — empty output +3. `go test -p 1 -count=1 ./...` — all pass, 0 skip +4. `scripts/smoke.sh` — ALL SMOKE TESTS PASSED +5. `scripts/smoke-web.sh` — passes (if compose available) +6. Baseline test: old DB → new code → `schema_migrations` has `0001_baseline.sql` row +7. No `CountAdmins` exported (removed) +8. No `ListBookIDs` exported (removed, dead code) +9. No local `isUnique` functions outside `store.IsUniqueViolation` +10. No `inside` function in scanner (replaced by `bookfile.Contains`) +11. Router contract test pins all routes +12. All 17 bugs (B1–B17) have corresponding tests or smoke verification diff --git a/docs/superpowers/specs/2026-09-14-backend-hardening-design.md b/docs/superpowers/specs/2026-09-14-backend-hardening-design.md new file mode 100644 index 0000000..d19a861 --- /dev/null +++ b/docs/superpowers/specs/2026-09-14-backend-hardening-design.md @@ -0,0 +1,160 @@ +# 后端健壮性(迁移系统 + bug 修复 + 结构重构/接口化)设计 / Backend hardening design + +日期 2026-09-14。分支 `fix/backend-hardening`。状态:已获用户批准(会话内确认)。 + +本 spec 是项目优化四个子项目中的 **①**(其余:② 阅读器改版、③ 功能增强、④ 前端工程质量,各自独立 spec)。③ 依赖本子项目先行(功能变更需要迁移机制)。 + +## 目标 / Goal + +1. schema 变更从「静默不生效」变为安全的有序迁移(多副本并发安全)。 +2. 修复探索阶段确认的 17 个缺陷(B1–B17),每项先有复现测试。 +3. 业务逻辑按 AGENTS.md 要求从 `cmd/webui/handlers` 下沉到 `internal/`,并以消费方小接口 + 手写 fake 实现可脱库单测。 +4. 引入 CI workflow 文件(GitHub Actions 语法,兼容 Gitea Actions),runner 就绪前本地门禁为强制。 + +非目标见文末「范围外」。 + +## 决策记录 / Decisions + +- (a)迁移 + (b)bug + (c)重构全做,按 a→b→c 分批实现、分批提交。用户选定。 +- 迁移机制选**自研极简版**(embedded SQL + `schema_migrations` + pg advisory lock),否决 golang-migrate/goose(单人项目、5 张表,依赖+CLI 工作流偏重)。用户选定。 +- **不支持 down 迁移**:回滚靠备份,fix-forward。决策写入 README。 +- 重构深度选**方案 2 = 下沉 + 全面接口化**。用户选定。约束原则(防止接口层变负资产): + - 接口按消费方需要定义成小口径(`internal/ports`),不做镜像整个 store 的胖接口; + - 构造函数仍返回具体类型,`main.go` 手写装配,不引 DI 框架; + - 测试 fake 全部手写 in-memory 实现,不引 testify/gomock。 +- 否决方案 3(最小触碰):留下 5 处缓存键手工同步与 3 套路径包含校验,正是本次要修的 bug 温床。 +- CI:远端为自托管 Gitea(`git.yoresee.cc`,暂无 runner)。workflow 按标准 GitHub Actions 语法写入 `.github/workflows/`——Gitea Actions 直接兼容,未来迁 GitHub 零改动;runner 就绪前每批合入前跑本地门禁。用户选定。 +- API 契约零变化,例外仅为 B6/B7/B8 的错误语义修正(记 changelog)。 + +## S1 迁移系统 / Migration system + +- `backend/internal/db/migrations/0001_baseline.sql` = 现 `schema.sql` 原样搬入;`schema.sql` 删除。后续变更只新增 `NNNN_描述.sql`(四位序号),**已应用的文件永不修改**。 +- `schema_migrations(version BIGINT PRIMARY KEY, name TEXT NOT NULL, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())`,建表语句内置于 `db.Migrate`(非迁移文件)。 +- `db.Migrate(ctx, pool)` 流程: + 1. `pg_advisory_lock(<固定 int64 常量,定义在 db 包>)`,defer 解锁——`--scale api=N` 副本串行化; + 2. `CREATE TABLE IF NOT EXISTS schema_migrations ...`; + 3. **基线检测**:`schema_migrations` 为空且 `to_regclass('books')` 非空 → 直接登记 0001 已应用,不重跑(老库原地升级); + 4. `go:embed migrations` 按文件名排序,逐个未应用版本在**独立事务**内 `exec 文件 + INSERT 登记`;任一失败:回滚该事务、日志报出版本号与错误、返回 error → `main` 非零退出(compose restart 兜底,fix-forward)。 +- 迁移文件校验:文件名必须匹配 `^\d{4}_[a-z0-9_]+\.sql$`,embed 列表里出现不合法名直接 panic(启动即失败,早于任何 DDL)。 + +## S2 bug 修复清单 / Bug fixes + +每项**先写复现测试(红)→ 修(绿)**。归属层接口化后能 fake 单测的单测,否则集成测试(真 PG/Redis)。批次 B 完成 B1–B13;B14/B15/B17 随批次 A(与迁移/启动路径相邻);B16 随批次 C(依赖 `internal/upload` 下沉)。依赖未就绪新包的两处例外:B9-② single-flight 随批次 C(依赖 scanner 重构),B9-① 锁续期在批次 B 于 redispkg 现体内实现;B8 在批次 B 预建 `internal/media` 包仅放保留名纯函数,批次 C 补全该包其余内容。 + +| # | 缺陷(现状文件:行) | 修法 | +|---|---|---| +| B1 | `redispkg.IncrWindow`:INCR 成功但 EXPIRE 失败 → key 永不过期,该 IP **永久限流**(redis.go:55-57) | Lua 脚本原子 `INCR`+`EXPIRE`(首值时设 TTL);redis 错误维持 fail-open 返回 1 | +| B2 | `redispkg.Lock`:`rand.Read` 错误被吞 → 全零 token 可被他人偷锁(redis.go:66-67) | rand 失败 → 记日志并按故障降级路径返回 `(noop, true)` | +| B3 | `Lock` 的 unlock 用调用方 ctx:取消/关停后 Eval 静默失败,锁挂满 5min TTL(redis.go:77-81) | unlock 内部改用 `context.WithoutCancel(ctx)`;Eval 失败记日志 | +| B4 | 分片 part 以 `O_TRUNC` 直写最终名:写一半崩溃 → 截断片被 `UploadStatus` 报「已收到」(uploads.go:202-206) | 写 `parts/N.tmp` + rename;`Complete` 的总长校验保留为第二道防线 | +| B5 | 删最后 admin 是 TOCTOU(两并发请求可删光 admin);`n, _ := CountAdmins` 吞 DB 错误 → 误导性 400(users.go:85-92) | 规则下沉 store:`DeleteUser` 内部同一事务做 last-admin 检查+删除,冲突返回哨兵 `store.ErrLastAdmin`;handler 映射 400,DB 错误 → 503。自我删除检查留在 handler | +| B6 | 上传 `io.Copy` 任何失败(磁盘满/断连)都报 `413 too_large`(libraries.go:149-153) | 仅 `errors.Is(err, http.MaxBytesError)` → 413;其余 → 500 | +| B7 | `/auth/me` 把所有 store 错误(含 PG 宕机)映射 401(handlers/auth.go:50-55) | 仅 no-rows → 401;其余走既有 `dbErr` | +| B8 | 库名可叫 `cache` / `.uploads`,与 `CACHE_DIR`、上传会话目录冲突(scanner 会走缓存树、SweepStale 会误删) | `POST /libraries` 拒绝保留名 → `400 reserved_name`;保留集常量定义在 `internal/media`(布局唯一事实源) | +| B9 | scan 锁 5min TTL 不续期(大库扫描时第二副本加入同一棵树);无 redis 时每次点扫描**无上限起 goroutine**(scanner.go:61 注释、libraries.go:103) | ① 续期封装进 `ScanLock` 实现:持锁期间每 TTL/2 自动续期,unlock 停止;② scanner 加**进程内 per-library single-flight**(同库并发触发合并为一次,无 redis 也生效) | +| B10 | scanner `SetBookState` 返回值丢弃 → 坏书静默保持 ready(scanner.go:160,183) | 记 error 日志(扫描继续,不中断整轮) | +| B11 | 封面写盘错误全静默、孤儿 `.tmp`、rename 成功后仍无条件 `os.Remove(tmp)`(scanner.go:224-228、content.go:60-71) | 收敛到 `internal/media.WriteAtomic`:错误全检查、全记日志,仅失败路径清 tmp | +| B12 | `uniquePath` stat-then-create 竞态:并发同名上传选中同一候选 → `O_EXCL` 失败 500(libraries.go:168-186) | create 冲突时重取候选名,有限次重试循环 | +| B13 | `store.go` 在检查 err 前读 `res.RowsAffected()`(store.go:359-365) | 调序(先 err 后 rows) | +| B14 | serve goroutine 内 `log.Fatalf` 绕过 defer/优雅关停(main.go:50) | `srv.ListenAndServe` 错误经 channel 交回 main,统一走 shutdown 路径退出 | +| B15 | `DATABASE_URL` 空/非法延迟到 pgxpool 才报晦涩错;`REDIS_URL` 空静默禁用全部防护(config.go:65-66) | `config.Load` 校验:DATABASE_URL 必填且可解析,fail-fast 带清晰消息;REDIS_URL 允许空但打日志「redis disabled: rate-limit/scan-lock/page-cache off」 | +| B16 | `sweepUploads` 同步跑在 `UploadInit` 请求路径里(uploads.go:137) | 移入 scanner ticker(每轮顺手清),请求路径不再做全盘 ReadDir | +| B17 | `scripts/smoke.sh` 仍 POST 被忽略的 `root_path`(smoke.sh:32) | 脚本对齐现契约(只发 `{name}`) | + +## S3 结构重构 + 接口化 / Restructure + +### 目标布局 + +``` +backend/ +├── cmd/webui/ +│ ├── main.go # 装配:具体实现 → ports 注入;启动/关停(含 B14) +│ ├── api/router.go # 路由表不变 + 新增契约测试 +│ └── handlers/ # 只剩 bind/validate/调端口/哨兵错误→HTTP 码 +├── internal/ +│ ├── ports/ # ★ 全部消费方接口 + 跨包哨兵错误重导出 +│ ├── store/ # 按聚合拆:users.go libraries.go books.go progress.go +│ │ # bookmarks.go store.go(类型/ctor/InTx);pool 收私有; +│ │ # 导出 IsUniqueViolation;删死码 ListBookIDs +│ ├── media/ # ★ 缓存布局唯一事实源 + 提取/章节(详下) +│ ├── upload/ # ★ 分片会话子系统全量下沉(详下) +│ ├── bookfile/ # + OpenReaderAt(合并 3 处 open+stat+fn 重复); +│ │ # + Contains(parent,child) 统一三套路径包含校验(EvalSymlinks 语义) +│ ├── scanner/ # add/update 合一为 ingest(persist 回调);single-flight;接管 B16 +│ ├── redispkg/ # 实现 PageCache/RateLimiter/ScanLock(Lua 原子化,B1-B3) +│ ├── auth/ config/ db/ seed/ # db+迁移系统;config+校验(B15);seed 走 ports +``` + +### ports 接口清单(方法集按现有具体实现机械映射,签名以 plan 为准) + +- `UserStore`:CountUsers / CreateUser / GetUserByName / GetUserByID / ListUsers / DeleteUser(含 B5 事务化 last-admin 规则,返回 `ErrLastAdmin`)。`CountAdmins` 从公开面消失。 +- `LibraryStore`:CreateLibrary / ListLibraries / GetLibrary。 +- `BookStore`:InsertBook / GetBook / ListBookMeta / UpdateBookFile / DeleteBookByPath / DeleteBook / SetBookState / ListBooks / BookHashes。 +- `ProgressStore`:UpsertProgress / GetProgress / ListProgress。 +- `BookmarkStore`:InsertBookmark / ListBookmarks / UpdateBookmarkNote / DeleteBookmark。 +- `PageCache`(消费方:media):Get(ctx,key) (string,bool) / Set(ctx,key,val,ttl)。 +- `RateLimiter`(消费方:auth handler):IncrWindow(ctx,key,ttl) int。 +- `ScanLock`(消费方:scanner):Lock(ctx,key,ttl) (unlock func(), ok bool),实现内部自动续期(B9)。 +- `UploadSessions`(消费方:handlers/uploads):Init / Status / PutPart / Complete / Sweep;哨兵 `ErrTooLarge` `ErrIncomplete` `ErrSizeMismatch` `ErrNotFound`。 +- `Media`(消费方:handlers/content、scanner):EnsureCover / EnsurePage / ChaptersOf / PageIndex / CacheBuster;哨兵 `ErrBrokenArchive`。 +- `Scanner`(消费方:handlers/libraries):ScanLibraryByID。 + +哨兵错误定义在所属实现包,`ports` 统一重导出供 handler `errors.Is` 映射;pg 错误分类收敛为 `store.IsUniqueViolation(err)` 单一谓词(替代 handlers.dbErr/users.isUnique/seed 三份拷贝),no-rows 判断维持 `errors.Is(err, pgx.ErrNoRows)`。 + +### internal/media(缓存与提取的唯一事实源) + +收拢目前散布在 scanner、handlers/content、handlers/books 的隐式共享知识: + +- 布局与键:`DirKey(id,size,modTS)`、`CoverDir`、`PagesDir`、`CacheBuster`(`?v=` hash)、保留名集合(B8)——5 处手工同步归一。保留名校验以**纯函数** `media.IsReservedName(name) bool` 暴露,libraries handler 直接 import 使用(无 I/O,不进 Media 接口、不需 fake)。 +- `WriteAtomic`:唯一 tmp+rename 实现(替代 5 处拷贝,B11)。 +- 提取:cbz/epub 封面、cbz 页(含自愈:磁盘缓存缺失时按需重建,现 content.go 的懒加载逻辑迁入);`ChaptersOf`(现 handlers 的 chaptersOf 纯域规则迁入);`PageIndex`(zip 索引 + redis 缓存策略,键 `pagesidx2:*` 不变)。 + +### internal/upload(分片会话子系统) + +现 handlers/uploads.go 全部 285 行域逻辑迁入:会话 id 派生(sha256 确定性)、目录布局(`/.uploads//{meta.json,parts/N}`)、分片校验(≤32MB、索引合法)、meta 读写(损坏 meta 记日志并按新会话处理,不再静默摧毁)、TTL 清理(Sweep,由 scanner ticker 调)、拼装+原子落盘+去重后缀(B12 的重试在此实现)。`UniquePath` 以导出函数住在 internal/upload,单发上传 handler 与分片拼装共用同一份。handler 只剩 JSON 绑定、调端口、哨兵→HTTP 码。 + +### handlers 去重(随下沉自然消除) + +- `getLibrary`/`getLibRow` 二合一;`ParseInt(c.Param("id"))` 样板 → 单一 `idParam(c)` helper。 +- 上传校验(SafeName+FormatFromExt+同一错误文案)单发/分片两路共用一份(住在 internal/upload)。 +- 路径包含校验统一 `bookfile.Contains`(三套实现收敛为 EvalSymlinks 语义一套)。 + +### 测试 + +- `api/router_test.go` 扩为**路由契约测试**:测试内 pin 一份期望路由表(golden 集合),遍历 gin `Routes()` 断言与之完全一致、admin 路由挂 AdminOnly 中间件——任何未过审的路由增删改都会红。 +- 新增 `handlers/*_unit_test.go`:fake(内存实现 ports)驱动,无 PG/Redis 可跑,覆盖哨兵→HTTP 码映射与 bind/validate 分支。 +- fake 统一住在 `internal/ports/portsfake` 一个共享包,手写、无生成器。 +- 现有 47 个集成测试全保留;测试助手去重(writeCBZ/testCfg/setup 收敛到共享测试包)。 + +## S4 CI 与验证 / CI & gates + +- `.github/workflows/ci.yml`(Gitea Actions 兼容语法): + - job `backend`:actions/checkout + actions/setup-go(版本读 go.mod)+ service 容器 `postgres:16`、`redis:7`;步骤:`gofmt -l .`(输出非空即败)、`go vet ./...`、`go test -p 1 -count=1 ./...`(注入 `DATABASE_URL`/`REDIS_URL` 指向 service,CI 中不存在 skip 路径)。 + - job `frontend`:actions/setup-node + `npm ci` + `npm run check`。 + - workflow 语法本地用 `actionlint`(`go run` 一次性执行,不入 go.mod)自检。 +- docs/README.md + README_zh.md(同步):重写「改 schema 前必读」为迁移工作流;新增 CI 节(如何在 Gitea 开启 Actions/注册 act_runner;迁 GitHub 零改动)。 +- **本地门禁(runner 就绪前强制)**:每批合入前 dev compose 起 PG+Redis,`go vet ./... && gofmt -l . && go test -p 1 -count=1 ./...` 确认 **0 skip**,再跑 `scripts/smoke.sh` + `scripts/smoke-web.sh`。 +- 批次 A 额外验证**老库基线路径**:先用当前 master 镜像建库建表,再换本分支启动,断言 `schema_migrations` 被基线为 0001 且无 DDL 重跑。 +- 分支 `fix/backend-hardening`;本 spec 与实现同分支提交。 + +## 实现批次 / Batches + +1. **A**:迁移系统(S1)+ CI workflow(S4)+ B14/B15/B17。门禁:本地全量 + 老库基线验证 + actionlint。 +2. **B**:B1–B13,每项复现测试先行。门禁:本地全量(新测试含 fake 前置形态,接口未拆前允许先以集成测试写就,批次 C 迁移为单测)。 +3. **C**:S3 全部(ports/store 拆分/media/upload/scanner/bookfile/handlers 瘦身)+ B16 + 契约测试 + fake 单测。纯结构、行为不变。门禁:本地全量 + smoke + 契约测试绿。 + +每批独立提交(`git commit` 粒度按聚合/主题),批内保持测试常绿;CHANGELOG 条目在对应批次落地时写入。 + +## 文档与 changelog 义务 / Docs + +- `docs/CHANGELOG.md`(非 WebUI,双语同条、条目间空行)至少记录:迁移系统(Changed)、B1 永久限流(Fixed)、B4 截断分片(Fixed)、B6 413 语义(Fixed)、B7 me 错误语义(Fixed)、B8 reserved_name(Changed);其余内部修复酌情合并一条。 +- README 双版同步(S4 所列两节)。 +- AGENTS.md 无需改动(本次是向它的规则收敛)。 + +## 范围外 / Out of scope (YAGNI) + +- down 迁移、迁移 CLI 工具化。 +- 列表分页、JWT 吊销/刷新、库重命名/删除、扫描状态 API、元数据编辑——子项目 ③。 +- 任何前端改动——子项目 ②/④。 +- DI 框架、mock 生成器、`pkg/` 公开面。 +- 数据库层性能(连接池参数调优、索引审计)——无证据表明当前是瓶颈。 diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 92816a6..025ee88 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -29,7 +29,7 @@ code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "$API/users" "${J[@]}" -H say "library + bad-ext upload rejected + good upload + scan" mkdir -p deploy/api/storage/smoke-books -LID=$(curl -sf "$API/libraries" "${J[@]}" -H "$AUTH" -d '{"name":"smoke","root_path":"/data/books/smoke-books"}' | sed -E 's/.*"id":([0-9]+).*/\1/' || true) +LID=$(curl -sf "$API/libraries" "${J[@]}" -H "$AUTH" -d '{"name":"smoke"}' | sed -E 's/.*"id":([0-9]+).*/\1/' || true) [ -n "$LID" ] || LID=$(curl -fsS "$API/libraries" -H "$AUTH" | grep -oE '"id":[0-9]+,"name":"smoke"' | cut -d: -f2 | cut -d, -f1) [ -n "$LID" ] || die "no library id" printf 'x' > "$WORK/f"