feat(upload): resumable chunked upload protocol for large files — init(parts≤32MB)/status/complete endpoints with fingerprint-derived deterministic uploadId (same file resumes, no restart), sessions in BOOKS_DIR/.uploads with 24h sweep, frontend uploadBook auto-switches >16MB to 8MB parts (retry x3 in place, skip received); oversize single POST now reports honest 413 too_large instead of "multipart field 'file' required" (was MaxBytesReader abort mislabeled); UPLOAD_MAX_MB wired into both compose stacks (cap total, .env.example 2048), nginx body limit drops to 32m; testCfg limit 1MB, CJK/fullwidth-colon filename proven unaffected; e2e verified with 311MB real file (39 parts, sha1 match)
This commit is contained in:
@@ -0,0 +1,157 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func initUpload(t *testing.T, h http.Handler, tok, libID, name string, size, chunk int64) (map[string]any, *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
w := do(h, "POST", "/api/libraries/"+libID+"/upload/init", tok,
|
||||
map[string]any{"name": name, "size": size, "chunkSize": chunk})
|
||||
var m map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &m)
|
||||
return m, w
|
||||
}
|
||||
|
||||
func putPart(h http.Handler, tok, uid string, idx int, data []byte) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("PUT", "/api/uploads/"+uid+"/parts/"+fmt.Sprint(idx), bytes.NewReader(data))
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
ww := httptest.NewRecorder()
|
||||
h.ServeHTTP(ww, req)
|
||||
return ww
|
||||
}
|
||||
|
||||
func mkLib(t *testing.T, h http.Handler, tok, name string) string {
|
||||
t.Helper()
|
||||
w := do(h, "POST", "/api/libraries", tok, map[string]string{"name": name})
|
||||
if w.Code != 201 {
|
||||
t.Fatalf("create lib %d %s", w.Code, w.Body)
|
||||
}
|
||||
var m map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &m)
|
||||
return itoa(m["id"])
|
||||
}
|
||||
|
||||
func TestUploadChunkedHappyPath(t *testing.T) {
|
||||
_, _, h, booksDir := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
|
||||
content := bytes.Repeat([]byte("調教開關第二季!"), 40000) // ~880KB, <1MB 测试上限
|
||||
size := int64(len(content))
|
||||
chunk := int64(400000)
|
||||
m, w := initUpload(t, h, tok, libID, "調教開關:第二季.zip", size, chunk)
|
||||
if w.Code != 200 || m["uploadId"] == "" {
|
||||
t.Fatalf("init %d %s", w.Code, w.Body)
|
||||
}
|
||||
uid := m["uploadId"].(string)
|
||||
|
||||
n := int((size + chunk - 1) / chunk) // 乱序上传
|
||||
for _, i := range []int{2, 0, 1} {
|
||||
lo, hi := int64(i)*chunk, int64(i+1)*chunk
|
||||
if hi > size {
|
||||
hi = size
|
||||
}
|
||||
if ww := putPart(h, tok, uid, i, content[lo:hi]); ww.Code != 202 {
|
||||
t.Fatalf("part %d: %d %s", i, ww.Code, ww.Body)
|
||||
}
|
||||
}
|
||||
w = do(h, "GET", "/api/uploads/"+uid, tok, nil)
|
||||
var st struct{ Received []int }
|
||||
json.Unmarshal(w.Body.Bytes(), &st)
|
||||
if len(st.Received) != n {
|
||||
t.Fatalf("status want %d got %v", n, st.Received)
|
||||
}
|
||||
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
|
||||
if w.Code != 202 {
|
||||
t.Fatalf("complete %d %s", w.Code, w.Body)
|
||||
}
|
||||
var res map[string]any
|
||||
json.Unmarshal(w.Body.Bytes(), &res)
|
||||
if res["path"] != "調教開關:第二季.zip" {
|
||||
t.Fatalf("path: %v", res["path"])
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(booksDir, "s2", "調教開關:第二季.zip"))
|
||||
if err != nil || !bytes.Equal(got, content) {
|
||||
t.Fatalf("assembled file wrong: err=%v eq=%v", err, bytes.Equal(got, content))
|
||||
}
|
||||
if entries, _ := os.ReadDir(filepath.Join(booksDir, ".uploads")); len(entries) != 0 {
|
||||
t.Fatalf("session not cleaned: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunkedResumeKeepsParts(t *testing.T) {
|
||||
_, _, h, _ := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
content := bytes.Repeat([]byte("x"), 900000)
|
||||
m, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
|
||||
uid := m["uploadId"].(string)
|
||||
putPart(h, tok, uid, 0, content[:400000])
|
||||
// 同指纹重复 init → 复用会话,已传分片保留
|
||||
m2, _ := initUpload(t, h, tok, libID, "r.cbz", 900000, 400000)
|
||||
if m2["uploadId"] != uid {
|
||||
t.Fatalf("resume want same uid got %v", m2["uploadId"])
|
||||
}
|
||||
w := do(h, "GET", "/api/uploads/"+uid, tok, nil)
|
||||
if !strings.Contains(w.Body.String(), "[0]") {
|
||||
t.Fatalf("resumed status: %s", w.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunkedErrors(t *testing.T) {
|
||||
_, _, h, _ := setupAPI(t)
|
||||
tok := adminToken(t, h)
|
||||
libID := mkLib(t, h, tok, "s2")
|
||||
|
||||
// 总量超 UPLOAD_MAX_MB(测试=1MB)→ 413,消息带限额
|
||||
_, w := initUpload(t, h, tok, libID, "big.cbz", 2<<20, 400000)
|
||||
if w.Code != 413 || !strings.Contains(w.Body.String(), "too_large") {
|
||||
t.Fatalf("oversize want 413 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// 扩展名白名单
|
||||
_, w = initUpload(t, h, tok, libID, "virus.exe", 100, 10)
|
||||
if w.Code != 400 || !strings.Contains(w.Body.String(), "bad_format") {
|
||||
t.Fatalf("bad ext want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// chunkSize 超 32MB
|
||||
_, w = initUpload(t, h, tok, libID, "ok.cbz", 100, 40<<20)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("huge chunk want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
m, _ := initUpload(t, h, tok, libID, "p.cbz", 1000, 400)
|
||||
uid := m["uploadId"].(string)
|
||||
// 越界 index
|
||||
if ww := putPart(h, tok, uid, 9, bytes.Repeat([]byte("y"), 400)); ww.Code != 400 {
|
||||
t.Fatalf("index oob want 400 got %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
// 分片超期望体积
|
||||
if ww := putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 500)); ww.Code != 413 {
|
||||
t.Fatalf("part too big want 413 got %d %s", ww.Code, ww.Body)
|
||||
}
|
||||
// 缺片 complete
|
||||
putPart(h, tok, uid, 0, bytes.Repeat([]byte("y"), 400))
|
||||
w = do(h, "POST", "/api/uploads/"+uid+"/complete", tok, nil)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("missing parts want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
// 未知 uid → 404
|
||||
w = do(h, "GET", "/api/uploads/deadbeefdeadbeefdeadbeefdeadbeef", tok, nil)
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("unknown uid want 404 got %d", w.Code)
|
||||
}
|
||||
// 非法 uid → 400
|
||||
w = do(h, "GET", "/api/uploads/zzz", tok, nil)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("bad uid want 400 got %d %s", w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user