feat(upload): resumable chunked upload protocol for large files — init(parts≤32MB)/status/complete endpoints with fingerprint-derived deterministic uploadId (same file resumes, no restart), sessions in BOOKS_DIR/.uploads with 24h sweep, frontend uploadBook auto-switches >16MB to 8MB parts (retry x3 in place, skip received); oversize single POST now reports honest 413 too_large instead of "multipart field 'file' required" (was MaxBytesReader abort mislabeled); UPLOAD_MAX_MB wired into both compose stacks (cap total, .env.example 2048), nginx body limit drops to 32m; testCfg limit 1MB, CJK/fullwidth-colon filename proven unaffected; e2e verified with 311MB real file (39 parts, sha1 match)

This commit is contained in:
2026-09-07 23:56:40 +08:00
parent aba1e58fa6
commit 7d650107d2
12 changed files with 540 additions and 10 deletions
+157
View File
@@ -0,0 +1,157 @@
package handlers_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
func initUpload(t *testing.T, h http.Handler, tok, libID, name string, size, chunk int64) (map[string]any, *httptest.ResponseRecorder) {
t.Helper()
w := do(h, "POST", "/api/libraries/"+libID+"/upload/init", tok,
map[string]any{"name": name, "size": size, "chunkSize": chunk})
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return m, w
}
func putPart(h http.Handler, tok, uid string, idx int, data []byte) *httptest.ResponseRecorder {
req := httptest.NewRequest("PUT", "/api/uploads/"+uid+"/parts/"+fmt.Sprint(idx), bytes.NewReader(data))
req.Header.Set("Authorization", "Bearer "+tok)
req.Header.Set("Content-Type", "application/octet-stream")
ww := httptest.NewRecorder()
h.ServeHTTP(ww, req)
return ww
}
func mkLib(t *testing.T, h http.Handler, tok, name string) string {
t.Helper()
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": name})
if w.Code != 201 {
t.Fatalf("create lib %d %s", w.Code, w.Body)
}
var m map[string]any
json.Unmarshal(w.Body.Bytes(), &m)
return itoa(m["id"])
}
func TestUploadChunkedHappyPath(t *testing.T) {
_, _, h, booksDir := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("調教開關第二季!"), 40000) // ~880KB, <1MB 测试上限
size := int64(len(content))
chunk := int64(400000)
m, w := initUpload(t, h, tok, libID, "調教開關:第二季.zip", size, chunk)
if w.Code != 200 || m["uploadId"] == "" {
t.Fatalf("init %d %s", w.Code, w.Body)
}
uid := m["uploadId"].(string)
n := int((size + chunk - 1) / chunk) // 乱序上传
for _, i := range []int{2, 0, 1} {
lo, hi := int64(i)*chunk, int64(i+1)*chunk
if hi > size {
hi = size
}
if ww := putPart(h, tok, uid, i, content[lo:hi]); ww.Code != 202 {
t.Fatalf("part %d: %d %s", i, ww.Code, ww.Body)
}
}
w = do(h, "GET", "/api/uploads/"+uid, tok, nil)
var st struct{ Received []int }
json.Unmarshal(w.Body.Bytes(), &st)
if len(st.Received) != n {
t.Fatalf("status want %d got %v", n, st.Received)
}
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 202 {
t.Fatalf("complete %d %s", w.Code, w.Body)
}
var res map[string]any
json.Unmarshal(w.Body.Bytes(), &res)
if res["path"] != "調教開關:第二季.zip" {
t.Fatalf("path: %v", res["path"])
}
got, err := os.ReadFile(filepath.Join(booksDir, "s2", "調教開關:第二季.zip"))
if err != nil || !bytes.Equal(got, content) {
t.Fatalf("assembled file wrong: err=%v eq=%v", err, bytes.Equal(got, content))
}
if entries, _ := os.ReadDir(filepath.Join(booksDir, ".uploads")); len(entries) != 0 {
t.Fatalf("session not cleaned: %v", entries)
}
}
func TestUploadChunkedResumeKeepsParts(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
content := bytes.Repeat([]byte("x"), 900000)
m, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
uid := m["uploadId"].(string)
putPart(h, tok, uid, 0, content[:400000])
// 同指纹重复 init → 复用会话,已传分片保留
m2, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
if m2["uploadId"] != uid {
t.Fatalf("resume want same uid got %v", m2["uploadId"])
}
w := do(h, "GET", "/api/uploads/"+uid, tok, nil)
if !strings.Contains(w.Body.String(), "[0]") {
t.Fatalf("resumed status: %s", w.Body)
}
}
func TestUploadChunkedErrors(t *testing.T) {
_, _, h, _ := setupAPI(t)
tok := adminToken(t, h)
libID := mkLib(t, h, tok, "s2")
// 总量超 UPLOAD_MAX_MB(测试=1MB)→ 413,消息带限额
_, w := initUpload(t, h, tok, libID, "big.cbz", 2<<20, 400000)
if w.Code != 413 || !strings.Contains(w.Body.String(), "too_large") {
t.Fatalf("oversize want 413 got %d %s", w.Code, w.Body)
}
// 扩展名白名单
_, w = initUpload(t, h, tok, libID, "virus.exe", 100, 10)
if w.Code != 400 || !strings.Contains(w.Body.String(), "bad_format") {
t.Fatalf("bad ext want 400 got %d %s", w.Code, w.Body)
}
// chunkSize 超 32MB
_, w = initUpload(t, h, tok, libID, "ok.cbz", 100, 40<<20)
if w.Code != 400 {
t.Fatalf("huge chunk want 400 got %d %s", w.Code, w.Body)
}
m, _ := initUpload(t, h, tok, libID, "p.cbz", 1000, 400)
uid := m["uploadId"].(string)
// 越界 index
if ww := putPart(h, tok, uid, 9, bytes.Repeat([]byte("y"), 400)); ww.Code != 400 {
t.Fatalf("index oob want 400 got %d %s", ww.Code, ww.Body)
}
// 分片超期望体积
if ww := putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 500)); ww.Code != 413 {
t.Fatalf("part too big want 413 got %d %s", ww.Code, ww.Body)
}
// 缺片 complete
putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 400))
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
if w.Code != 400 {
t.Fatalf("missing parts want 400 got %d %s", w.Code, w.Body)
}
// 未知 uid → 404
w = do(h, "GET", "/api/uploads/deadbeefdeadbeefdeadbeefdeadbeef", tok, nil)
if w.Code != 404 {
t.Fatalf("unknown uid want 404 got %d", w.Code)
}
// 非法 uid → 400
w = do(h, "GET", "/api/uploads/zzz", tok, nil)
if w.Code != 400 {
t.Fatalf("bad uid want 400 got %d %s", w.Code, w.Body)
}
}