feat(backend): directory scanner — diff, error state, covers, cache sweep, redis lock
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"booklib/internal/bookfile"
|
||||
"booklib/internal/config"
|
||||
"booklib/internal/redispkg"
|
||||
"booklib/internal/store"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
st *store.Store
|
||||
cfg *config.Config
|
||||
rdb *redispkg.R
|
||||
}
|
||||
|
||||
func New(st *store.Store, cfg *config.Config, rdb *redispkg.R) *Scanner {
|
||||
return &Scanner{st: st, cfg: cfg, rdb: rdb}
|
||||
}
|
||||
|
||||
func (s *Scanner) Run(ctx context.Context) {
|
||||
t := time.NewTicker(s.cfg.ScanInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
libs, err := s.st.ListLibraries(ctx)
|
||||
if err != nil {
|
||||
log.Printf("scan: list libraries: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, l := range libs {
|
||||
s.ScanLibrary(ctx, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) ScanLibraryByID(ctx context.Context, id int64) {
|
||||
lib, err := s.st.GetLibrary(ctx, id)
|
||||
if err != nil {
|
||||
log.Printf("scan: library %d: %v", id, err)
|
||||
return
|
||||
}
|
||||
s.ScanLibrary(ctx, lib)
|
||||
}
|
||||
|
||||
func (s *Scanner) ScanLibrary(ctx context.Context, lib store.Library) {
|
||||
// ponytail: 5min lock TTL; a scan longer than this lets another replica join — refresh mid-walk if libs ever outgrow it
|
||||
unlock, ok := s.rdb.Lock(ctx, fmt.Sprintf("scan:%d", lib.ID), 5*time.Minute)
|
||||
if !ok {
|
||||
return // 别的副本在扫
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
root, err := filepath.EvalSymlinks(filepath.Clean(lib.RootPath))
|
||||
if err != nil || !inside(s.cfg.BooksDir, root) {
|
||||
log.Printf("scan: library %d root %q rejected", lib.ID, lib.RootPath)
|
||||
return
|
||||
}
|
||||
disk, err := walk(root)
|
||||
if err != nil {
|
||||
log.Printf("scan: walk %s: %v", root, err)
|
||||
return
|
||||
}
|
||||
dbMeta, err := s.st.ListBookMeta(ctx, lib.ID)
|
||||
if err != nil {
|
||||
log.Printf("scan: list books: %v", err)
|
||||
return
|
||||
}
|
||||
for rel, ds := range disk {
|
||||
old, exists := dbMeta[rel]
|
||||
delete(dbMeta, rel)
|
||||
switch {
|
||||
case !exists:
|
||||
s.add(ctx, lib.ID, root, rel, ds)
|
||||
case old.Size != ds.size || old.ModTS != ds.modTS:
|
||||
s.update(ctx, lib.ID, old.ID, root, rel, ds)
|
||||
}
|
||||
}
|
||||
for rel := range dbMeta { // 只剩被删的文件
|
||||
if err := s.st.DeleteBookByPath(ctx, lib.ID, rel); err != nil {
|
||||
log.Printf("scan: delete %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
s.sweepCache(ctx)
|
||||
}
|
||||
|
||||
type diskStat struct{ size, modTS int64 }
|
||||
|
||||
func inside(booksDir, root string) bool {
|
||||
b := filepath.Clean(booksDir)
|
||||
return root == b || strings.HasPrefix(root, b+string(os.PathSeparator))
|
||||
}
|
||||
|
||||
func walk(root string) (map[string]diskStat, error) {
|
||||
out := map[string]diskStat{}
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
log.Printf("scan: walk %s: %v", p, err)
|
||||
return nil // 单点失败不中断
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if bookfile.FormatFromExt(d.Name()) == "" {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[filepath.ToSlash(rel)] = diskStat{info.Size(), info.ModTime().Unix()}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func titleOf(rel string) string {
|
||||
base := filepath.Base(rel)
|
||||
return strings.TrimSpace(strings.ReplaceAll(strings.TrimSuffix(base, filepath.Ext(base)), "_", " "))
|
||||
}
|
||||
|
||||
// cbz 完整性判定集中在 add/update:PageIndex 失败 → state=error。
|
||||
// InsertBook/UpdateBookFile 的 SQL 已把 state 重置为 ready(Task 2),无需显式清 error。
|
||||
func (s *Scanner) add(ctx context.Context, libID int64, root, rel string, ds diskStat) {
|
||||
format := bookfile.FormatFromExt(filepath.Base(rel))
|
||||
pageCount := 0
|
||||
var idxErr error
|
||||
if format == "cbz" {
|
||||
idx, err := s.zipIndex(root, rel)
|
||||
pageCount = len(idx)
|
||||
idxErr = err
|
||||
}
|
||||
id, err := s.st.InsertBook(ctx, libID, rel, titleOf(rel), format, ds.size, ds.modTS, pageCount)
|
||||
if err != nil {
|
||||
log.Printf("scan: insert %s: %v", rel, err)
|
||||
return
|
||||
}
|
||||
if idxErr != nil {
|
||||
s.st.SetBookState(ctx, id, "error", idxErr.Error())
|
||||
return
|
||||
}
|
||||
s.cover(ctx, id, root, rel, format, ds)
|
||||
}
|
||||
|
||||
func (s *Scanner) update(ctx context.Context, libID, bookID int64, root, rel string, ds diskStat) {
|
||||
format := bookfile.FormatFromExt(filepath.Base(rel))
|
||||
pageCount := 0
|
||||
var idxErr error
|
||||
if format == "cbz" {
|
||||
idx, err := s.zipIndex(root, rel)
|
||||
pageCount = len(idx)
|
||||
idxErr = err
|
||||
}
|
||||
if err := s.st.UpdateBookFile(ctx, bookID, ds.size, ds.modTS, pageCount); err != nil {
|
||||
log.Printf("scan: update %s: %v", rel, err)
|
||||
return
|
||||
}
|
||||
if idxErr != nil {
|
||||
s.st.SetBookState(ctx, bookID, "error", idxErr.Error())
|
||||
return
|
||||
}
|
||||
s.cover(ctx, bookID, root, rel, format, ds)
|
||||
}
|
||||
|
||||
func (s *Scanner) zipIndex(root, rel string) ([]string, error) {
|
||||
f, err := os.Open(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bookfile.PageIndex(f, st.Size())
|
||||
}
|
||||
|
||||
// cover 失败(坏 epub、无图等)只 log — 书的 state 由 PageIndex 判定,封面缺了有占位 SVG 兜底
|
||||
func (s *Scanner) cover(ctx context.Context, id int64, root, rel, format string, ds diskStat) {
|
||||
var img []byte
|
||||
var ext string
|
||||
var err error
|
||||
switch format {
|
||||
case "cbz":
|
||||
img, ext, err = s.readCover(root, rel, bookfile.CBZCover)
|
||||
case "epub":
|
||||
img, ext, err = s.readCover(root, rel, bookfile.EPUBCover)
|
||||
default:
|
||||
return // pdf/txt/md 用占位 SVG,不落盘
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("scan: cover %s: %v", rel, err)
|
||||
return
|
||||
}
|
||||
dir := bookfile.CoverDir(s.cfg.CacheDir, bookfile.DirKey(id, bookfile.Hash(ds.size, ds.modTS)))
|
||||
if e := os.MkdirAll(dir, 0o755); e != nil {
|
||||
log.Printf("scan: coverdir %s: %v", rel, e)
|
||||
return
|
||||
}
|
||||
tmp := filepath.Join(dir, "cover"+ext+".tmp")
|
||||
dst := filepath.Join(dir, "cover"+ext)
|
||||
if e := os.WriteFile(tmp, img, 0o644); e == nil {
|
||||
os.Rename(tmp, dst)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) readCover(root, rel string, fn func(io.ReaderAt, int64) ([]byte, string, error)) ([]byte, string, error) {
|
||||
f, err := os.Open(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer f.Close()
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return fn(f, st.Size())
|
||||
}
|
||||
|
||||
func (s *Scanner) sweepCache(ctx context.Context) {
|
||||
hashes, err := s.st.BookHashes(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
live := map[string]bool{}
|
||||
for id, v := range hashes {
|
||||
live[bookfile.DirKey(id, bookfile.Hash(v[0], v[1]))] = true
|
||||
}
|
||||
if n, err := bookfile.SweepStale(s.cfg.CacheDir, live); err != nil {
|
||||
log.Printf("scan: sweep: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("scan: swept %d stale cache dirs", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user