feat(upload): resumable chunked upload protocol for large files — init(parts≤32MB)/status/complete endpoints with fingerprint-derived deterministic uploadId (same file resumes, no restart), sessions in BOOKS_DIR/.uploads with 24h sweep, frontend uploadBook auto-switches >16MB to 8MB parts (retry x3 in place, skip received); oversize single POST now reports honest 413 too_large instead of "multipart field 'file' required" (was MaxBytesReader abort mislabeled); UPLOAD_MAX_MB wired into both compose stacks (cap total, .env.example 2048), nginx body limit drops to 32m; testCfg limit 1MB, CJK/fullwidth-colon filename proven unaffected; e2e verified with 311MB real file (39 parts, sha1 match)

This commit is contained in:
2026-09-07 23:56:40 +08:00
parent aba1e58fa6
commit 7d650107d2
12 changed files with 540 additions and 10 deletions
+1 -1
View File
@@ -24,7 +24,7 @@ import (
)
func testCfg() *config.Config {
return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 200,
return &config.Config{Addr: ":8080", JWTSecret: []byte("s3cret"), ScanInterval: time.Minute, UploadMaxMB: 1, // 测试里 1MB:超限用例只需 2MB body
TrustedProxies: []string{"172.16.0.0/12"}} // 与 prod 默认一致: 只有 compose 网段内代理才可信
}
+5
View File
@@ -116,6 +116,11 @@ func (h *H) Upload(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.cfg.UploadMaxMB<<20)
fh, e := c.FormFile("file")
if e != nil {
var mbe *http.MaxBytesError
if errors.As(e, &mbe) {
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
return
}
err(c, http.StatusBadRequest, "bad_request", "multipart field 'file' required")
return
}
@@ -80,6 +80,41 @@ func TestLibraryCreateListUpload(t *testing.T) {
}
}
func TestUploadSizeAndFilename(t *testing.T) {
_, _, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": "s2"})
if w.Code != 201 {
t.Fatalf("create lib %d %s", w.Code, w.Body)
}
var lib map[string]any
json.Unmarshal(w.Body.Bytes(), &lib)
libID := itoa(lib["id"])
// 全角冒号等非 ASCII 文件名正常落盘
body, mw := uploadBody("調教開關:第二季.cbz", []byte("zipbytes"))
req := httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+tok)
ww := httptest.NewRecorder()
h.ServeHTTP(ww, req)
if ww.Code != 202 {
t.Fatalf("cjk name upload %d %s", ww.Code, ww.Body)
}
if _, err := os.Stat(filepath.Join(booksDir, "s2", "調教開關:第二季.cbz")); err != nil {
t.Fatal("cjk upload missing:", err)
}
// 超过 UploadMaxMB(测试=1MB)→ 413 too_large,而非误报 "file required"
body, mw = uploadBody("big.cbz", bytes.Repeat([]byte("x"), 2<<20))
req = httptest.NewRequest("POST", "/api/libraries/"+libID+"/upload", body)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+tok)
ww = httptest.NewRecorder()
h.ServeHTTP(ww, req)
if ww.Code != 413 || !strings.Contains(ww.Body.String(), "too_large") {
t.Fatalf("oversize want 413 too_large got %d %s", ww.Code, ww.Body)
}
}
func uploadBody(filename string, content []byte) (*bytes.Buffer, *multipart.Writer) {
buf := &bytes.Buffer{}
mw := multipart.NewWriter(buf)
+285
View File
@@ -0,0 +1,285 @@
package handlers
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"booklib/internal/bookfile"
)
// 分片上传:init(指纹→确定性 uploadId,天然支持续传)→ PUT parts → complete 拼接原子落盘。
// 会话即 <BooksDir>/.uploads/<uid>/(meta.json + parts/N),无独立状态存储;24h 未 complete opportunistic 清扫。
const (
maxChunkBytes = 32 << 20
defaultChunk = 8 << 20
uploadSessTTL = 24 * time.Hour
uploadSessionIn = ".uploads"
)
type uploadMeta struct {
Name string `json:"name"`
Size int64 `json:"size"`
ChunkSize int64 `json:"chunkSize"`
LibraryID int64 `json:"libraryId"`
}
func validUploadID(s string) bool {
if len(s) != 32 {
return false
}
for _, r := range s {
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f')) {
return false
}
}
return true
}
func uploadIDFor(libID int64, name string, size, chunk int64) string {
h := sha256.Sum256([]byte(fmt.Sprintf("%d|%s|%d|%d", libID, name, size, chunk)))
return hex.EncodeToString(h[:16])
}
func (h *H) uploadDir(uid string) string {
return filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn, uid)
}
func chunkRange(m uploadMeta, i int64) (int64, int64) {
lo := i * m.ChunkSize
hi := min(lo+m.ChunkSize, m.Size)
return lo, hi
}
func (h *H) numParts(m uploadMeta) int64 {
return (m.Size + m.ChunkSize - 1) / m.ChunkSize
}
// sweepUploads 删除过期会话目录;尽力而为,失败不影响主流程
func (h *H) sweepUploads() {
base := filepath.Join(filepath.Clean(h.cfg.BooksDir), uploadSessionIn)
es, e := os.ReadDir(base)
if e != nil {
return
}
for _, en := range es {
if fi, e := en.Info(); e == nil && time.Since(fi.ModTime()) > uploadSessTTL {
os.RemoveAll(filepath.Join(base, en.Name()))
}
}
}
func (h *H) UploadInit(c *gin.Context) {
lib, ok := h.getLibrary(c)
if !ok {
return
}
if _, ok := h.libRoot(c, lib); !ok {
return
}
var req struct {
Name string `json:"name"`
Size int64 `json:"size"`
ChunkSize int64 `json:"chunkSize"`
}
if c.ShouldBindJSON(&req) != nil {
err(c, http.StatusBadRequest, "bad_request", "name, size required")
return
}
name := bookfile.SafeName(req.Name)
if name == "" {
err(c, http.StatusBadRequest, "bad_request", "bad name")
return
}
if bookfile.FormatFromExt(name) == "" {
err(c, http.StatusBadRequest, "bad_format", "extension must be cbz/pdf/epub/txt/md")
return
}
if req.Size <= 0 {
err(c, http.StatusBadRequest, "bad_request", "bad size")
return
}
if req.Size > h.cfg.UploadMaxMB<<20 {
err(c, http.StatusRequestEntityTooLarge, "too_large", "file exceeds upload limit of "+strconv.FormatInt(h.cfg.UploadMaxMB, 10)+"MB")
return
}
if req.ChunkSize == 0 {
req.ChunkSize = defaultChunk
}
if req.ChunkSize > maxChunkBytes {
err(c, http.StatusBadRequest, "bad_request", "chunkSize must be <= 33554432")
return
}
uid := uploadIDFor(lib.ID, name, req.Size, req.ChunkSize)
dir := h.uploadDir(uid)
meta := uploadMeta{Name: name, Size: req.Size, ChunkSize: req.ChunkSize, LibraryID: lib.ID}
if b, e := os.ReadFile(filepath.Join(dir, "meta.json")); e == nil {
var old uploadMeta
if json.Unmarshal(b, &old) == nil && old == meta { // 同指纹 → 复用会话(续传)
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
return
}
os.RemoveAll(dir) // 指纹撞上但内容不同 → 从头再来
}
h.sweepUploads()
if e := os.MkdirAll(filepath.Join(dir, "parts"), 0o755); e != nil {
err(c, http.StatusInternalServerError, "internal", "create session")
return
}
b, _ := json.Marshal(meta)
if e := os.WriteFile(filepath.Join(dir, "meta.json"), b, 0o644); e != nil {
err(c, http.StatusInternalServerError, "internal", "write meta")
return
}
c.JSON(http.StatusOK, gin.H{"uploadId": uid})
}
// loadMeta 校验 uid 与路径,404/400 已回复
func (h *H) loadMeta(c *gin.Context) (uploadMeta, string, bool) {
uid := c.Param("uid")
if !validUploadID(uid) {
err(c, http.StatusBadRequest, "bad_request", "bad upload id")
return uploadMeta{}, "", false
}
dir := h.uploadDir(uid)
var m uploadMeta
b, e := os.ReadFile(filepath.Join(dir, "meta.json"))
if e != nil {
err(c, http.StatusNotFound, "not_found", "no such upload")
return uploadMeta{}, "", false
}
if json.Unmarshal(b, &m) != nil {
err(c, http.StatusInternalServerError, "internal", "corrupt session")
return uploadMeta{}, "", false
}
return m, dir, true
}
func (h *H) UploadStatus(c *gin.Context) {
_, dir, ok := h.loadMeta(c)
if !ok {
return
}
recv := []int64{}
es, e := os.ReadDir(filepath.Join(dir, "parts"))
if e == nil {
for _, en := range es {
if i, e := strconv.ParseInt(en.Name(), 10, 64); e == nil {
recv = append(recv, i)
}
}
}
sort.Slice(recv, func(i, j int) bool { return recv[i] < recv[j] })
c.JSON(http.StatusOK, gin.H{"received": recv})
}
func (h *H) UploadPart(c *gin.Context) {
m, dir, ok := h.loadMeta(c)
if !ok {
return
}
idx, e := strconv.ParseInt(c.Param("index"), 10, 64)
if e != nil || idx < 0 || idx >= h.numParts(m) {
err(c, http.StatusBadRequest, "bad_request", "bad part index")
return
}
lo, hi := chunkRange(m, idx)
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, hi-lo)
p := filepath.Join(dir, "parts", strconv.FormatInt(idx, 10))
f, e := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if e != nil {
err(c, http.StatusInternalServerError, "internal", "create part")
return
}
n, e := io.Copy(f, c.Request.Body)
f.Close()
if e != nil || n != hi-lo {
os.Remove(p)
var mbe *http.MaxBytesError
code, msg := "too_large", "part size mismatch"
if errors.As(e, &mbe) {
msg = "part exceeds declared size"
}
err(c, http.StatusRequestEntityTooLarge, code, msg)
return
}
c.JSON(http.StatusAccepted, gin.H{"accepted": true})
}
func (h *H) UploadComplete(c *gin.Context) {
m, dir, ok := h.loadMeta(c)
if !ok {
return
}
var total int64
for i := int64(0); i < h.numParts(m); i++ {
lo, hi := chunkRange(m, i)
fi, e := os.Stat(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
if e != nil || fi.Size() != hi-lo {
err(c, http.StatusBadRequest, "bad_request", "upload incomplete; missing or corrupt parts, re-upload them")
return
}
total += fi.Size()
}
if total != m.Size {
err(c, http.StatusBadRequest, "bad_request", "total size mismatch")
return
}
lib, e := h.st.GetLibrary(c, m.LibraryID)
if e != nil {
dbErr(c, e)
return
}
root, ok := h.libRoot(c, lib)
if !ok {
return
}
dst, e := h.uniquePath(root, m.Name)
if e != nil {
err(c, http.StatusForbidden, "forbidden", e.Error())
return
}
tmp := filepath.Join(dir, "assembled")
out, e := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if e != nil {
err(c, http.StatusInternalServerError, "internal", "create tmp")
return
}
for i := int64(0); i < h.numParts(m); i++ {
pf, e := os.Open(filepath.Join(dir, "parts", strconv.FormatInt(i, 10)))
if e != nil {
out.Close()
err(c, http.StatusInternalServerError, "internal", "open part")
return
}
if _, e := io.Copy(out, pf); e != nil {
pf.Close()
out.Close()
os.Remove(tmp)
err(c, http.StatusInternalServerError, "internal", "assemble")
return
}
pf.Close()
}
out.Close()
if e := os.Rename(tmp, dst); e != nil { // 原子落盘,scanner 自动收编
os.Remove(tmp)
err(c, http.StatusInternalServerError, "internal", "rename")
return
}
os.RemoveAll(dir)
c.JSON(http.StatusAccepted, gin.H{"accepted": true, "path": strings.TrimPrefix(dst, root+string(os.PathSeparator))})
}
+157
View File
@@ -0,0 +1,157 @@
package handlers_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
func initUpload(t *testing.T, h http.Handler, tok, libID, name string, size, chunk int64) (map[string]any, *httptest.ResponseRecorder) {
t.Helper()
w := do(h, "POST", "/api/libraries/"+libID+"/upload/init", tok,
map[string]any{"name": name, "size": size, "chunkSize": chunk})
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return m, w
}
func putPart(h http.Handler, tok, uid string, idx int, data []byte) *httptest.ResponseRecorder {
req := httptest.NewRequest("PUT", "/api/uploads/"+uid+"/parts/"+fmt.Sprint(idx), bytes.NewReader(data))
req.Header.Set("Authorization", "Bearer "+tok)
req.Header.Set("Content-Type", "application/octet-stream")
ww := httptest.NewRecorder()
h.ServeHTTP(ww, req)
return ww
}
func mkLib(t *testing.T, h http.Handler, tok, name string) string {
t.Helper()
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": name})
if w.Code != 201 {
t.Fatalf("create lib %d %s", w.Code, w.Body)
}
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return itoa(m["id"])
}
func TestUploadChunkedHappyPath(t *testing.T) {
_, _, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("調教開關第二季!"), 40000) // ~880KB, <1MB 测试上限
size := int64(len(content))
chunk := int64(400000)
m, w := initUpload(t, h, tok, libID, "調教開關:第二季.zip", size, chunk)
if w.Code != 200 || m["uploadId"] == "" {
t.Fatalf("init %d %s", w.Code, w.Body)
}
uid := m["uploadId"].(string)
n := int((size + chunk - 1) / chunk) // 乱序上传
for _, i := range []int{2, 0, 1} {
lo, hi := int64(i)*chunk, int64(i+1)*chunk
if hi > size {
hi = size
}
if ww := putPart(h, tok, uid, i, content[lo:hi]); ww.Code != 202 {
t.Fatalf("part %d: %d %s", i, ww.Code, ww.Body)
}
}
w = do(h, "GET", "/api/uploads/"+uid, tok, nil)
var st struct{ Received []int }
json.Unmarshal(w.Body.Bytes(), &st)
if len(st.Received) != n {
t.Fatalf("status want %d got %v", n, st.Received)
}
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 202 {
t.Fatalf("complete %d %s", w.Code, w.Body)
}
var res map[string]any
json.Unmarshal(w.Body.Bytes(), &res)
if res["path"] != "調教開關:第二季.zip" {
t.Fatalf("path: %v", res["path"])
}
got, err := os.ReadFile(filepath.Join(booksDir, "s2", "調教開關:第二季.zip"))
if err != nil || !bytes.Equal(got, content) {
t.Fatalf("assembled file wrong: err=%v eq=%v", err, bytes.Equal(got, content))
}
if entries, _ := os.ReadDir(filepath.Join(booksDir, ".uploads")); len(entries) != 0 {
t.Fatalf("session not cleaned: %v", entries)
}
}
func TestUploadChunkedResumeKeepsParts(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("x"), 900000)
m, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
uid := m["uploadId"].(string)
putPart(h, tok, uid, 0, content[:400000])
// 同指纹重复 init → 复用会话,已传分片保留
m2, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
if m2["uploadId"] != uid {
t.Fatalf("resume want same uid got %v", m2["uploadId"])
}
w := do(h, "GET", "/api/uploads/"+uid, tok, nil)
if !strings.Contains(w.Body.String(), "[0]") {
t.Fatalf("resumed status: %s", w.Body)
}
}
func TestUploadChunkedErrors(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
// 总量超 UPLOAD_MAX_MB(测试=1MB)→ 413,消息带限额
_, w := initUpload(t, h, tok, libID, "big.cbz", 2<<20, 400000)
if w.Code != 413 || !strings.Contains(w.Body.String(), "too_large") {
t.Fatalf("oversize want 413 got %d %s", w.Code, w.Body)
}
// 扩展名白名单
_, w = initUpload(t, h, tok, libID, "virus.exe", 100, 10)
if w.Code != 400 || !strings.Contains(w.Body.String(), "bad_format") {
t.Fatalf("bad ext want 400 got %d %s", w.Code, w.Body)
}
// chunkSize 超 32MB
_, w = initUpload(t, h, tok, libID, "ok.cbz", 100, 40<<20)
if w.Code != 400 {
t.Fatalf("huge chunk want 400 got %d %s", w.Code, w.Body)
}
m, _ := initUpload(t, h, tok, libID, "p.cbz", 1000, 400)
uid := m["uploadId"].(string)
// 越界 index
if ww := putPart(h, tok, uid, 9, bytes.Repeat([]byte("y"), 400)); ww.Code != 400 {
t.Fatalf("index oob want 400 got %d %s", ww.Code, ww.Body)
}
// 分片超期望体积
if ww := putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 500)); ww.Code != 413 {
t.Fatalf("part too big want 413 got %d %s", ww.Code, ww.Body)
}
// 缺片 complete
putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 400))
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 400 {
t.Fatalf("missing parts want 400 got %d %s", w.Code, w.Body)
}
// 未知 uid → 404
w = do(h, "GET", "/api/uploads/deadbeefdeadbeefdeadbeefdeadbeef", tok, nil)
if w.Code != 404 {
t.Fatalf("unknown uid want 404 got %d", w.Code)
}
// 非法 uid → 400
w = do(h, "GET", "/api/uploads/zzz", tok, nil)
if w.Code != 400 {
t.Fatalf("bad uid want 400 got %d %s", w.Code, w.Body)
}
}