Merge branch 'feat/chunked-upload'
This commit is contained in:
@@ -37,6 +37,12 @@ func NewRouter(cfg *config.Config, st *store.Store, rdb *redispkg.R, sc *scanner
|
||||
libs.POST("", h.AdminOnly(), h.CreateLibrary)
|
||||
libs.POST("/:id/scan", h.AdminOnly(), h.ScanLibrary)
|
||||
libs.POST("/:id/upload", h.AdminOnly(), h.Upload)
|
||||
libs.POST("/:id/upload/init", h.AdminOnly(), h.UploadInit)
|
||||
|
||||
uploads := p.Group("/uploads", h.AdminOnly())
|
||||
uploads.GET("/:uid", h.UploadStatus)
|
||||
uploads.POST("/:uid/complete", h.UploadComplete)
|
||||
uploads.PUT("/:uid/parts/:index", h.UploadPart)
|
||||
|
||||
p.GET("/books", h.ListBooks)
|
||||
p.GET("/books/:id", h.GetBook)
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
)
|
||||
|
||||
func testCfg() *config.Config {
|
||||
return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 200,
|
||||
return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 1, // 测试里 1MB:超限用例只需 2MB body
|
||||
TrustedProxies: []string{"172.16.0.0/12"}} // 与 prod 默认一致: 只有 compose 网段内代理才可信
|
||||
}
|
||||
|
||||
|
||||
@@ -116,6 +116,11 @@ func (h *H) Upload(c *gin.Context) {
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.cfg.UploadMaxMB<<20)
|
||||
fh, e := c.FormFile("file")
|
||||
if e != nil {
|
||||
var mbe *http.MaxBytesError
|
||||
if errors.As(e, &mbe) {
|
||||
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
|
||||
return
|
||||
}
|
||||
err(c, http.StatusBadRequest, "bad_request", "multipart field 'file' required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -80,6 +80,41 @@ func TestLibraryCreateListUpload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSizeAndFilename(t *testing.T) {
|
||||
_, _, h, booksDir := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": "s2"})
|
||||
if w.Code != 201 {
|
||||
t.Fatalf("create lib %d %s", w.Code, w.Body)
|
||||
}
|
||||
var lib map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &lib)
|
||||
libID := itoa(lib["id"])
|
||||
// 全角冒号等非 ASCII 文件名正常落盘
|
||||
body, mw := uploadBody("調教開關:第二季.cbz", []byte("zipbytes"))
|
||||
req := httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
ww := httptest.NewRecorder()
|
||||
h.ServeHTTP(ww, req)
|
||||
if ww.Code != 202 {
|
||||
t.Fatalf("cjk name upload %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(booksDir, "s2", "調教開關:第二季.cbz")); err != nil {
|
||||
t.Fatal("cjk upload missing:", err)
|
||||
}
|
||||
// 超过 UploadMaxMB(测试=1MB)→ 413 too_large,而非误报 "file required"
|
||||
body, mw = uploadBody("big.cbz", bytes.Repeat([]byte("x"), 2<<20))
|
||||
req = httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
ww = httptest.NewRecorder()
|
||||
h.ServeHTTP(ww, req)
|
||||
if ww.Code != 413 || !strings.Contains(ww.Body.String(), "too_large") {
|
||||
t.Fatalf("oversize want 413 too_large got %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadBody(filename string, content []byte) (*bytes.Buffer, *multipart.Writer) {
|
||||
buf := &bytes.Buffer{}
|
||||
mw := multipart.NewWriter(buf)
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"booklib/internal/bookfile"
|
||||
)
|
||||
|
||||
// 分片上传:init(指纹→确定性 uploadId,天然支持续传)→ PUT parts → complete 拼接原子落盘。
|
||||
// 会话即 <BooksDir>/.uploads/<uid>/(meta.json + parts/N),无独立状态存储;24h 未 complete opportunistic 清扫。
|
||||
|
||||
const (
|
||||
maxChunkBytes = 32 << 20
|
||||
defaultChunk = 8 << 20
|
||||
uploadSessTTL = 24 * time.Hour
|
||||
uploadSessionIn = ".uploads"
|
||||
)
|
||||
|
||||
type uploadMeta struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
LibraryID int64 `json:"libraryId"`
|
||||
}
|
||||
|
||||
func validUploadID(s string) bool {
|
||||
if len(s) != 32 {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func uploadIDFor(libID int64, name string, size, chunk int64) string {
|
||||
h := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%d|%d", libID, name, size, chunk)))
|
||||
return hex.EncodeToString(h[:16])
|
||||
}
|
||||
|
||||
func (h *H) uploadDir(uid string) string {
|
||||
return filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn, uid)
|
||||
}
|
||||
|
||||
func chunkRange(m uploadMeta, i int64) (int64, int64) {
|
||||
lo := i * m.ChunkSize
|
||||
hi := min(lo+m.ChunkSize, m.Size)
|
||||
return lo, hi
|
||||
}
|
||||
|
||||
func (h *H) numParts(m uploadMeta) int64 {
|
||||
return (m.Size + m.ChunkSize - 1) / m.ChunkSize
|
||||
}
|
||||
|
||||
// sweepUploads 删除过期会话目录;尽力而为,失败不影响主流程
|
||||
func (h *H) sweepUploads() {
|
||||
base := filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn)
|
||||
es, e := os.ReadDir(base)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
for _, en := range es {
|
||||
if fi, e := en.Info(); e == nil && time.Since(fi.ModTime()) > uploadSessTTL {
|
||||
os.RemoveAll(filepath.Join(base, en.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *H) UploadInit(c *gin.Context) {
|
||||
lib, ok := h.getLibrary(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, ok := h.libRoot(c, lib); !ok {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
ChunkSize int64 `json:"chunkSize"`
|
||||
}
|
||||
if c.ShouldBindJSON(&req) != nil {
|
||||
err(c, http.StatusBadRequest, "bad_request", "name, size required")
|
||||
return
|
||||
}
|
||||
name := bookfile.SafeName(req.Name)
|
||||
if name == "" {
|
||||
err(c, http.StatusBadRequest, "bad_request", "bad name")
|
||||
return
|
||||
}
|
||||
if bookfile.FormatFromExt(name) == "" {
|
||||
err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md")
|
||||
return
|
||||
}
|
||||
if req.Size <= 0 {
|
||||
err(c, http.StatusBadRequest, "bad_request", "bad size")
|
||||
return
|
||||
}
|
||||
if req.Size > h.cfg.UploadMaxMB<<20 {
|
||||
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
|
||||
return
|
||||
}
|
||||
if req.ChunkSize == 0 {
|
||||
req.ChunkSize = defaultChunk
|
||||
}
|
||||
if req.ChunkSize > maxChunkBytes {
|
||||
err(c, http.StatusBadRequest, "bad_request", "chunkSize must be <= 33554432")
|
||||
return
|
||||
}
|
||||
uid := uploadIDFor(lib.ID, name, req.Size, req.ChunkSize)
|
||||
dir := h.uploadDir(uid)
|
||||
meta := uploadMeta{Name: name, Size: req.Size, ChunkSize: req.ChunkSize, LibraryID: lib.ID}
|
||||
if b, e := os.ReadFile(filepath.Join(dir, "meta.json")); e == nil {
|
||||
var old uploadMeta
|
||||
if json.Unmarshal(b, &old) == nil && old == meta { // 同指纹 → 复用会话(续传)
|
||||
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
|
||||
return
|
||||
}
|
||||
os.RemoveAll(dir) // 指纹撞上但内容不同 → 从头再来
|
||||
}
|
||||
h.sweepUploads()
|
||||
if e := os.MkdirAll(filepath.Join(dir, "parts"), 0o755); e != nil {
|
||||
err(c, http.StatusInternalServerError, "internal", "create session")
|
||||
return
|
||||
}
|
||||
b, _ := json.Marshal(meta)
|
||||
if e := os.WriteFile(filepath.Join(dir, "meta.json"), b, 0o644); e != nil {
|
||||
err(c, http.StatusInternalServerError, "internal", "write meta")
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
|
||||
}
|
||||
|
||||
// loadMeta 校验 uid 与路径,404/400 已回复
|
||||
func (h *H) loadMeta(c *gin.Context) (uploadMeta, string, bool) {
|
||||
uid := c.Param("uid")
|
||||
if !validUploadID(uid) {
|
||||
err(c, http.StatusBadRequest, "bad_request", "bad upload id")
|
||||
return uploadMeta{}, "", false
|
||||
}
|
||||
dir := h.uploadDir(uid)
|
||||
var m uploadMeta
|
||||
b, e := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||
if e != nil {
|
||||
err(c, http.StatusNotFound, "not_found", "no such upload")
|
||||
return uploadMeta{}, "", false
|
||||
}
|
||||
if json.Unmarshal(b, &m) != nil {
|
||||
err(c, http.StatusInternalServerError, "internal", "corrupt session")
|
||||
return uploadMeta{}, "", false
|
||||
}
|
||||
return m, dir, true
|
||||
}
|
||||
|
||||
func (h *H) UploadStatus(c *gin.Context) {
|
||||
_, dir, ok := h.loadMeta(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
recv := []int64{}
|
||||
es, e := os.ReadDir(filepath.Join(dir, "parts"))
|
||||
if e == nil {
|
||||
for _, en := range es {
|
||||
if i, e := strconv.ParseInt(en.Name(), 10, 64); e == nil {
|
||||
recv = append(recv, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(recv, func(i, j int) bool { return recv[i] < recv[j] })
|
||||
c.JSON(http.StatusOK, gin.H{"received": recv})
|
||||
}
|
||||
|
||||
func (h *H) UploadPart(c *gin.Context) {
|
||||
m, dir, ok := h.loadMeta(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
idx, e := strconv.ParseInt(c.Param("index"), 10, 64)
|
||||
if e != nil || idx < 0 || idx >= h.numParts(m) {
|
||||
err(c, http.StatusBadRequest, "bad_request", "bad part index")
|
||||
return
|
||||
}
|
||||
lo, hi := chunkRange(m, idx)
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, hi-lo)
|
||||
p := filepath.Join(dir, "parts", strconv.FormatInt(idx, 10))
|
||||
f, e := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if e != nil {
|
||||
err(c, http.StatusInternalServerError, "internal", "create part")
|
||||
return
|
||||
}
|
||||
n, e := io.Copy(f, c.Request.Body)
|
||||
f.Close()
|
||||
if e != nil || n != hi-lo {
|
||||
os.Remove(p)
|
||||
var mbe *http.MaxBytesError
|
||||
code, msg := "too_large", "part size mismatch"
|
||||
if errors.As(e, &mbe) {
|
||||
msg = "part exceeds declared size"
|
||||
}
|
||||
err(c, http.StatusRequestEntityTooLarge, code, msg)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusAccepted, gin.H{"accepted": true})
|
||||
}
|
||||
|
||||
func (h *H) UploadComplete(c *gin.Context) {
|
||||
m, dir, ok := h.loadMeta(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var total int64
|
||||
for i := int64(0); i < h.numParts(m); i++ {
|
||||
lo, hi := chunkRange(m, i)
|
||||
fi, e := os.Stat(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
|
||||
if e != nil || fi.Size() != hi-lo {
|
||||
err(c, http.StatusBadRequest, "bad_request", "upload incomplete; missing or corrupt parts, re-upload them")
|
||||
return
|
||||
}
|
||||
total += fi.Size()
|
||||
}
|
||||
if total != m.Size {
|
||||
err(c, http.StatusBadRequest, "bad_request", "total size mismatch")
|
||||
return
|
||||
}
|
||||
lib, e := h.st.GetLibrary(c, m.LibraryID)
|
||||
if e != nil {
|
||||
dbErr(c, e)
|
||||
return
|
||||
}
|
||||
root, ok := h.libRoot(c, lib)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
dst, e := h.uniquePath(root, m.Name)
|
||||
if e != nil {
|
||||
err(c, http.StatusForbidden, "forbidden", e.Error())
|
||||
return
|
||||
}
|
||||
tmp := filepath.Join(dir, "assembled")
|
||||
out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if e != nil {
|
||||
err(c, http.StatusInternalServerError, "internal", "create tmp")
|
||||
return
|
||||
}
|
||||
for i := int64(0); i < h.numParts(m); i++ {
|
||||
pf, e := os.Open(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
|
||||
if e != nil {
|
||||
out.Close()
|
||||
err(c, http.StatusInternalServerError, "internal", "open part")
|
||||
return
|
||||
}
|
||||
if _, e := io.Copy(out, pf); e != nil {
|
||||
pf.Close()
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
err(c, http.StatusInternalServerError, "internal", "assemble")
|
||||
return
|
||||
}
|
||||
pf.Close()
|
||||
}
|
||||
out.Close()
|
||||
if e := os.Rename(tmp, dst); e != nil { // 原子落盘,scanner 自动收编
|
||||
os.Remove(tmp)
|
||||
err(c, http.StatusInternalServerError, "internal", "rename")
|
||||
return
|
||||
}
|
||||
os.RemoveAll(dir)
|
||||
c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": strings.TrimPrefix(dst, root+string(os.PathSeparator))})
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func initUpload(t *testing.T, h http.Handler, tok, libID, name string, size, chunk int64) (map[string]any, *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
w := do(h, "POST", "/api/libraries/"+libID+"/upload/init", tok,
|
||||
map[string]any{"name": name, "size": size, "chunkSize": chunk})
|
||||
var m map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &m)
|
||||
return m, w
|
||||
}
|
||||
|
||||
func putPart(h http.Handler, tok, uid string, idx int, data []byte) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("PUT", "/api/uploads/"+uid+"/parts/"+fmt.Sprint(idx), bytes.NewReader(data))
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
ww := httptest.NewRecorder()
|
||||
h.ServeHTTP(ww, req)
|
||||
return ww
|
||||
}
|
||||
|
||||
func mkLib(t *testing.T, h http.Handler, tok, name string) string {
|
||||
t.Helper()
|
||||
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": name})
|
||||
if w.Code != 201 {
|
||||
t.Fatalf("create lib %d %s", w.Code, w.Body)
|
||||
}
|
||||
var m map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &m)
|
||||
return itoa(m["id"])
|
||||
}
|
||||
|
||||
func TestUploadChunkedHappyPath(t *testing.T) {
|
||||
_, _, h, booksDir := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
|
||||
content := bytes.Repeat([]byte("調教開關第二季!"), 40000) // ~880KB, <1MB 测试上限
|
||||
size := int64(len(content))
|
||||
chunk := int64(400000)
|
||||
m, w := initUpload(t, h, tok, libID, "調教開關:第二季.zip", size, chunk)
|
||||
if w.Code != 200 || m["uploadId"] == "" {
|
||||
t.Fatalf("init %d %s", w.Code, w.Body)
|
||||
}
|
||||
uid := m["uploadId"].(string)
|
||||
|
||||
n := int((size + chunk - 1) / chunk) // 乱序上传
|
||||
for _, i := range []int{2, 0, 1} {
|
||||
lo, hi := int64(i)*chunk, int64(i+1)*chunk
|
||||
if hi > size {
|
||||
hi = size
|
||||
}
|
||||
if ww := putPart(h, tok, uid, i, content[lo:hi]); ww.Code != 202 {
|
||||
t.Fatalf("part %d: %d %s", i, ww.Code, ww.Body)
|
||||
}
|
||||
}
|
||||
w = do(h, "GET", "/api/uploads/"+uid, tok, nil)
|
||||
var st struct{ Received []int }
|
||||
json.Unmarshal(w.Body.Bytes(), &st)
|
||||
if len(st.Received) != n {
|
||||
t.Fatalf("status want %d got %v", n, st.Received)
|
||||
}
|
||||
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
|
||||
if w.Code != 202 {
|
||||
t.Fatalf("complete %d %s", w.Code, w.Body)
|
||||
}
|
||||
var res map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &res)
|
||||
if res["path"] != "調教開關:第二季.zip" {
|
||||
t.Fatalf("path: %v", res["path"])
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(booksDir, "s2", "調教開關:第二季.zip"))
|
||||
if err != nil || !bytes.Equal(got, content) {
|
||||
t.Fatalf("assembled file wrong: err=%v eq=%v", err, bytes.Equal(got, content))
|
||||
}
|
||||
if entries, _ := os.ReadDir(filepath.Join(booksDir, ".uploads")); len(entries) != 0 {
|
||||
t.Fatalf("session not cleaned: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunkedResumeKeepsParts(t *testing.T) {
|
||||
_, _, h, _ := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
content := bytes.Repeat([]byte("x"), 900000)
|
||||
m, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
|
||||
uid := m["uploadId"].(string)
|
||||
putPart(h, tok, uid, 0, content[:400000])
|
||||
// 同指纹重复 init → 复用会话,已传分片保留
|
||||
m2, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
|
||||
if m2["uploadId"] != uid {
|
||||
t.Fatalf("resume want same uid got %v", m2["uploadId"])
|
||||
}
|
||||
w := do(h, "GET", "/api/uploads/"+uid, tok, nil)
|
||||
if !strings.Contains(w.Body.String(), "[0]") {
|
||||
t.Fatalf("resumed status: %s", w.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunkedErrors(t *testing.T) {
|
||||
_, _, h, _ := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
|
||||
// 总量超 UPLOAD_MAX_MB(测试=1MB)→ 413,消息带限额
|
||||
_, w := initUpload(t, h, tok, libID, "big.cbz", 2<<20, 400000)
|
||||
if w.Code != 413 || !strings.Contains(w.Body.String(), "too_large") {
|
||||
t.Fatalf("oversize want 413 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// 扩展名白名单
|
||||
_, w = initUpload(t, h, tok, libID, "virus.exe", 100, 10)
|
||||
if w.Code != 400 || !strings.Contains(w.Body.String(), "bad_format") {
|
||||
t.Fatalf("bad ext want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// chunkSize 超 32MB
|
||||
_, w = initUpload(t, h, tok, libID, "ok.cbz", 100, 40<<20)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("huge chunk want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
m, _ := initUpload(t, h, tok, libID, "p.cbz", 1000, 400)
|
||||
uid := m["uploadId"].(string)
|
||||
// 越界 index
|
||||
if ww := putPart(h, tok, uid, 9, bytes.Repeat([]byte("y"), 400)); ww.Code != 400 {
|
||||
t.Fatalf("index oob want 400 got %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
// 分片超期望体积
|
||||
if ww := putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 500)); ww.Code != 413 {
|
||||
t.Fatalf("part too big want 413 got %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
// 缺片 complete
|
||||
putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 400))
|
||||
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("missing parts want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// 未知 uid → 404
|
||||
w = do(h, "GET", "/api/uploads/deadbeefdeadbeefdeadbeefdeadbeef", tok, nil)
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("unknown uid want 404 got %d", w.Code)
|
||||
}
|
||||
// 非法 uid → 400
|
||||
w = do(h, "GET", "/api/uploads/zzz", tok, nil)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("bad uid want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -2,10 +2,12 @@ JWT_SECRET=change-me-openssl-rand-hex-32
|
||||
ADMIN_USER=admin
|
||||
ADMIN_PASSWORD=change-me-min-8
|
||||
SCAN_INTERVAL_SEC=60
|
||||
# 上传总大小上限(MB)。前端 >16MB 自动分片(每片 8MB),nginx 体积限制只需盖住单个分片
|
||||
UPLOAD_MAX_MB=2048
|
||||
|
||||
# ---- 部署参数(deploy/prepare.sh 渲染模板 / compose 插值用) ----
|
||||
WEB_PORT=8080
|
||||
NGINX_CLIENT_MAX_BODY_SIZE=200m
|
||||
NGINX_CLIENT_MAX_BODY_SIZE=32m
|
||||
REDIS_MAXMEMORY=128mb
|
||||
REDIS_MAXMEMORY_POLICY=allkeys-lru
|
||||
DELVE_PORT=2345
|
||||
|
||||
@@ -28,6 +28,7 @@ services:
|
||||
BOOKS_DIR: /data/books
|
||||
CACHE_DIR: /data/books/cache
|
||||
SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60}
|
||||
UPLOAD_MAX_MB: ${UPLOAD_MAX_MB:-200}
|
||||
volumes:
|
||||
- ../backend:/app
|
||||
- ./api/storage:/data/books
|
||||
|
||||
@@ -22,6 +22,7 @@ services:
|
||||
BOOKS_DIR: /data/books
|
||||
CACHE_DIR: /data/books/cache
|
||||
SCAN_INTERVAL_SEC: ${SCAN_INTERVAL_SEC:-60}
|
||||
UPLOAD_MAX_MB: ${UPLOAD_MAX_MB:-200}
|
||||
volumes:
|
||||
- ./api/storage:/data/books
|
||||
depends_on:
|
||||
|
||||
@@ -8,8 +8,15 @@ The format loosely follows Keep a Changelog and can be adapted to the team's hab
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added / 新增
|
||||
|
||||
- API: resumable chunked upload protocol for large files — `POST /api/libraries/:id/upload/init` (fingerprint-derived deterministic `uploadId`, rejects totals over `UPLOAD_MAX_MB` with `413 too_large`), `PUT /api/uploads/:uid/parts/:index` (parts ≤ 32MB), `GET /api/uploads/:uid` (received parts, for resume), `POST /api/uploads/:uid/complete` (assemble + atomic land, same path contract as single-POST upload). Sessions persist under `BOOKS_DIR/.uploads/` with 24h opportunistic sweep. `UPLOAD_MAX_MB` is now wired through both compose stacks/`.env`; `.env.example` sets 2048 and drops `NGINX_CLIENT_MAX_BODY_SIZE` to 32m (nginx only ever sees one chunk).
|
||||
- API: 新增大文件可续传分片上传协议——`POST /api/libraries/:id/upload/init`(按指纹派生确定性 `uploadId`,总量超 `UPLOAD_MAX_MB` 返回 `413 too_large`)、`PUT /api/uploads/:uid/parts/:index`(单片 ≤32MB)、`GET /api/uploads/:uid`(查询已传分片以续传)、`POST /api/uploads/:uid/complete`(拼接后原子落盘,返回与单发上传一致的 `path`)。会话存于 `BOOKS_DIR/.uploads/`,超 24h 顺手清理。`UPLOAD_MAX_MB` 已接入两份 compose/`.env`;`.env.example` 调至 2048 并将 `NGINX_CLIENT_MAX_BODY_SIZE` 降为 32m(nginx 只见单个分片)。
|
||||
|
||||
### Changed / 变更
|
||||
|
||||
- API: upload over `UPLOAD_MAX_MB` now returns `413 too_large` with the limit in the message; previously the size abort was misreported as `400 bad_request "multipart field 'file' required"`.
|
||||
- API:超过 `UPLOAD_MAX_MB` 的上传现在返回 `413 too_large` 并在消息中带上限额;此前体积超限被误报为 `400 bad_request "multipart field 'file' required"`。
|
||||
- API: `POST /api/libraries` now takes only `{name}`; `root_path` is generated server-side as `BOOKS_DIR/<sanitized name>` (no client-supplied paths, validated at creation).
|
||||
- API:`POST /api/libraries` 只需 `{name}`;`root_path` 由服务端生成为 `BOOKS_DIR/<清洗后的库名>`(不再接受客户端指定路径,创建时即校验)。
|
||||
- Repo structure conformed to `AGENTS.md`: `web/` renamed to `frontend/`; backend HTTP layer moved from `internal/api` to `cmd/webui/{api,handlers}` (`cmd/server` → `cmd/webui`); README/CHANGELOGs relocated under `docs/` (`README_zh.md` added as Chinese mirror); module-level `.gitignore`s added (`backend/`, `deploy/`); stray root `library/` removed (book files live in `deploy/api/storage/`); debug binaries untracked.
|
||||
|
||||
@@ -10,6 +10,9 @@ The format loosely follows Keep a Changelog and can be adapted to the team's hab
|
||||
|
||||
### Added / 新增
|
||||
|
||||
- Library uploads over 16MB now auto-switch to the resumable chunked protocol (8MB parts): failed parts retry in place up to 3 times and already-sent parts are skipped, so a flaky upload continues instead of starting over; small files keep the original single request.
|
||||
- 书库上传超过 16MB 自动切换为可续传分片协议(每片 8MB):失败的分片原地重试至多 3 次,已传片自动跳过,中断后继续而不是从头再来;小文件仍走原单次上传。
|
||||
|
||||
- Immersive reading for text & comics (微信读书/Mihon-style): tap the middle of the page to show/hide the reader header and control bar; controls now live in a slide-up bottom sheet — a progress slider jumps to any position (text) or page (CBZ), theme swatches (纸/米/夜) are WYSIWYG color dots, plus A−/A+ font size; body text is serif with a readable justified measure; all prefs persist (localStorage).
|
||||
- 文本与漫画进入沉浸阅读:点正文中央唤出/隐藏顶栏与工具条;控件收进底部滑出抽屉——进度/页码滑条直接跳到全书任意位置或任意页,主题改为所见即所得的色卡(纸/米/夜),保留 A−/A+ 字号;txt/md 正文为衬线、限宽两端对齐;偏好本地持久化。
|
||||
- CBZ adds reading modes — 连读 (continuous scroll) / 整页 (snap to page top) / 适高 (one page per screen, full width) — cycle button in the bar, choice remembered; tapping the left/right edges turns a page/screen and ←/→/PageUp/PageDown work too.
|
||||
|
||||
@@ -113,6 +113,33 @@ export function formatPageUrl(fmt: string, n: number): string {
|
||||
return fmt.replace("%d", String(n));
|
||||
}
|
||||
|
||||
const SINGLE_MAX = 16 << 20;
|
||||
const CHUNK_SIZE = 8 << 20;
|
||||
const PART_RETRY = 3;
|
||||
|
||||
async function uploadChunked(id: number, file: File): Promise<{ accepted: boolean; path: string }> {
|
||||
const { uploadId } = await apiFetch<{ uploadId: string }>(`/libraries/${id}/upload/init`, {
|
||||
method: "POST",
|
||||
body: { name: file.name, size: file.size, chunkSize: CHUNK_SIZE },
|
||||
});
|
||||
const { received } = await apiFetch<{ received: number[] }>(`/uploads/${uploadId}`);
|
||||
const have = new Set(received);
|
||||
const n = Math.ceil(file.size / CHUNK_SIZE);
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (have.has(i)) continue;
|
||||
for (let tries = 0; ; tries++) {
|
||||
const res = await fetch(full(`/uploads/${uploadId}/parts/${i}`), {
|
||||
method: "PUT",
|
||||
headers: { ...authHeaders(), "Content-Type": "application/octet-stream" },
|
||||
body: file.slice(i * CHUNK_SIZE, Math.min((i + 1) * CHUNK_SIZE, file.size)),
|
||||
});
|
||||
if (res.ok) break;
|
||||
if (tries >= PART_RETRY) throw await toHttpError(res);
|
||||
}
|
||||
}
|
||||
return apiFetch<{ accepted: boolean; path: string }>(`/uploads/${uploadId}/complete`, { method: "POST" });
|
||||
}
|
||||
|
||||
export const api = {
|
||||
login: (username: string, password: string) =>
|
||||
apiFetch<{ token: string }>("/auth/login", { method: "POST", body: { username, password } }),
|
||||
@@ -126,14 +153,15 @@ export const api = {
|
||||
listLibraries: () => apiFetch<Library[]>("/libraries"),
|
||||
createLibrary: (name: string) => apiFetch<Library>("/libraries", { method: "POST", body: { name } }),
|
||||
scanLibrary: (id: number) => apiFetch<{ accepted: boolean }>(`/libraries/${id}/scan`, { method: "POST" }),
|
||||
uploadBook: (id: number, file: File) => {
|
||||
// ≤16MB 单 POST;更大走分片协议(init→parts→complete),失败分片原地重试,已传分片跳过
|
||||
uploadBook: (id: number, file: File): Promise<{ accepted: boolean; path: string }> =>
|
||||
file.size <= SINGLE_MAX
|
||||
? (() => {
|
||||
const fd = new FormData();
|
||||
fd.append("file", file);
|
||||
return apiFetch<{ accepted: boolean; path: string }>(`/libraries/${id}/upload`, {
|
||||
method: "POST",
|
||||
form: fd,
|
||||
});
|
||||
},
|
||||
return apiFetch<{ accepted: boolean; path: string }>(`/libraries/${id}/upload`, { method: "POST", form: fd });
|
||||
})()
|
||||
: uploadChunked(id, file),
|
||||
|
||||
listBooks: (params: { library?: number; q?: string } = {}) => {
|
||||
const sp = new URLSearchParams();
|
||||
|
||||
Reference in New Issue
Block a user