migrate from github

cmd/db_init/admin.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+func createUserInTx(tx *gorm.DB, username, email, password string) error {
+	logrus.Printf("Creating user %s in transaction...", username)
+
+	hashedPwd, err := utils.HashPassword(password)
+	if err != nil {
+		return err
+	}
+	externalID := utils.GenerateExternalID(utils.ExternalIDContextDocument)
+
+	user := model.User{
+		ExternalID:   externalID,
+		Username:     username,
+		PasswordHash: hashedPwd,
+		Email:        email,
+		Nickname:     username,
+		Status:       1,
+	}
+
+	var count int64
+	tx.Model(&model.User{}).Where("email = ?", user.Email).Count(&count)
+	if count == 0 {
+		if err := tx.Create(&user).Error; err != nil {
+			return err
+		}
+		logrus.Printf("User %s created successfully in transaction.", username)
+	} else {
+		logrus.Printf("User %s already exists in transaction.", username)
+	}
+
+	return nil
+}
+
+func createUserWithUsername(tx *gorm.DB, username string) error {
+	email := fmt.Sprintf("%s@yoresee.cc", username)
+	password := username
+	return createUserInTx(tx, username, email, password)
+}
+
+func createAdminUserInTx(tx *gorm.DB) error {
+	logrus.Println("Creating admin user in transaction...")
+
+	if err := createUserWithUsername(tx, "admin"); err != nil {
+		return err
+	}
+
+	logrus.Println("Admin permission granted successfully in transaction.")
+	return nil
+}

cmd/db_init/config.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"github.com/XingfenD/yoresee_doc/internal/constant"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+)
+
+func initializeConfigInConsul(ctx context.Context) error {
+	registerModeKey := utils.GenConfigKey(
+		constant.ConfigKey_First_System,
+		constant.ConfigKey_Second_Security,
+		constant.ConfigKey_Third_RegisterMode,
+	)
+
+	if _, ok, err := storage.Consul.Get(ctx, registerModeKey); err != nil {
+		return err
+	} else if !ok {
+		if err := storage.Consul.Set(ctx, registerModeKey, constant.RegisterMode_Invite); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func markDatabaseInitializedInConsul(ctx context.Context) error {
+	initializedKey := utils.GenConfigKey(
+		constant.ConfigKey_First_System,
+		constant.ConfigKey_Second_Database,
+		constant.ConfigKey_Third_Initialized,
+	)
+	return storage.Consul.Set(ctx, initializedKey, constant.Database_Initialized_True)
+}

cmd/db_init/document.go

@@ -0,0 +1,182 @@
+package main
+
+import (
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+type DocumentInput struct {
+	Title       string
+	Content     string
+	Summary     string
+	ParentID    int64
+	Tags        []string
+	KnowledgeID *int64
+}
+
+func createDocumentWithAllTables(tx *gorm.DB, adminUserID int64, input DocumentInput) (*model.Document, error) {
+	document := model.Document{
+		ExternalID:  utils.GenerateExternalID(utils.ExternalIDContextDocument),
+		Title:       input.Title,
+		Type:        "markdown",
+		Summary:     input.Summary,
+		ParentID:    input.ParentID,
+		UserID:      adminUserID,
+		Content:     input.Content,
+		Tags:        input.Tags,
+		ViewCount:   0,
+		EditCount:   0,
+		KnowledgeID: input.KnowledgeID,
+		Path:        "0",
+		Depth:       0,
+	}
+
+	if err := tx.Create(&document).Error; err != nil {
+		return nil, err
+	}
+	if err := updateDocumentPathDepth(tx, document.ID, document.ParentID); err != nil {
+		return nil, err
+	}
+
+	documentVersion := model.DocumentVersion{
+		DocumentID:    document.ID,
+		Version:       1,
+		Title:         document.Title,
+		Content:       document.Content,
+		UserID:        adminUserID,
+		ChangeSummary: "Initial version",
+	}
+
+	if err := tx.Create(&documentVersion).Error; err != nil {
+		return nil, err
+	}
+
+	return &document, nil
+}
+
+func initializeDocumentsInTx(tx *gorm.DB) error {
+	logrus.Println("Initializing default documents in transaction...")
+
+	var adminUser model.User
+	if err := tx.Where("username = ?", "admin").First(&adminUser).Error; err != nil {
+		return err
+	}
+
+	var count int64
+	tx.Model(&model.Document{}).Where("title = ?", "欢迎使用 Yoresee Doc").Count(&count)
+	if count > 0 {
+		logrus.Println("Default document already exists in transaction.")
+		return nil
+	}
+	contentString := "# 欢迎使用 Yoresee Doc\n\n这是您的第一个文档。Yoresee Doc 是一个功能强大的文档管理系统，支持以下特性：\n\n- 📝 富文本编辑\n- 📁 文档分类管理\n- 🔍 全文搜索\n- 👥 协作编辑\n- 📊 版本控制\n- 🔒 权限管理\n\n## 快速开始\n\n1. 点击左侧菜单创建新文档\n2. 使用编辑器撰写内容\n3. 保存文档并分享给团队成员\n\n祝您使用愉快！"
+
+	document := model.Document{
+		ExternalID: utils.GenerateExternalID(utils.ExternalIDContextDocument),
+		Title:      "欢迎使用 Yoresee Doc",
+		Type:       "markdown",
+		Summary:    "Yoresee Doc 系统欢迎文档",
+		ParentID:   0,
+		UserID:     adminUser.ID,
+		Content:    contentString,
+		Tags:       []string{"guide", "welcome"},
+		ViewCount:  0,
+		EditCount:  0,
+		Path:       "0",
+		Depth:      0,
+	}
+
+	if err := tx.Create(&document).Error; err != nil {
+		return err
+	}
+	if err := updateDocumentPathDepth(tx, document.ID, document.ParentID); err != nil {
+		return err
+	}
+
+	documentVersion := model.DocumentVersion{
+		DocumentID:    document.ID,
+		Version:       1,
+		Title:         document.Title,
+		Content:       contentString,
+		UserID:        adminUser.ID,
+		ChangeSummary: "Initial version",
+	}
+
+	if err := tx.Create(&documentVersion).Error; err != nil {
+		return err
+	}
+
+	logrus.Println("Adding child documents to welcome document...")
+
+	childDocumentTitles := []string{
+		"功能介绍",
+		"使用教程",
+		"常见问题",
+		"更新日志",
+		"联系我们",
+	}
+
+	for _, title := range childDocumentTitles {
+		childDoc, err := createDocumentWithAllTables(tx, adminUser.ID, DocumentInput{
+			Title:       title,
+			Content:     "# " + title + "\n\n这是" + title + "的详细文档内容。",
+			Summary:     title + "相关文档",
+			ParentID:    document.ID,
+			Tags:        []string{"welcome", title},
+			KnowledgeID: nil,
+		})
+		if err != nil {
+			logrus.Warnf("Failed to create child document %s: %v", title, err)
+			continue
+		}
+
+		for k := 1; k <= 2; k++ {
+			grandchildTitle := title + " - 详情" + string(rune('0'+k))
+			_, err := createDocumentWithAllTables(tx, adminUser.ID, DocumentInput{
+				Title:       grandchildTitle,
+				Content:     "# " + grandchildTitle + "\n\n这是" + grandchildTitle + "的详细文档内容。",
+				Summary:     grandchildTitle + "相关文档",
+				ParentID:    childDoc.ID,
+				Tags:        []string{"welcome", title, "详情"},
+				KnowledgeID: nil,
+			})
+			if err != nil {
+				logrus.Warnf("Failed to create grandchild document %s: %v", grandchildTitle, err)
+			}
+		}
+	}
+
+	logrus.Println("Child documents added to welcome document successfully")
+
+	logrus.Println("Default document created successfully in transaction")
+	return nil
+}
+
+func updateDocumentPathDepth(tx *gorm.DB, docID int64, parentID int64) error {
+	if parentID == 0 {
+		return tx.Exec(`
+			UPDATE document_metas
+			SET path = (id::text)::ltree, depth = 0
+			WHERE id = ?
+		`, docID).Error
+	}
+
+	type pathDepth struct {
+		Path  string
+		Depth int
+	}
+	var parent pathDepth
+	if err := tx.Model(&model.Document{}).
+		Select("path, depth").
+		Where("id = ?", parentID).
+		Take(&parent).Error; err != nil {
+		return err
+	}
+
+	return tx.Exec(`
+		UPDATE document_metas
+		SET path = (?::ltree) || (id::text)::ltree, depth = ?
+		WHERE id = ?
+	`, parent.Path, parent.Depth+1, docID).Error
+}

cmd/db_init/knowledge_base.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+func initializeKnowledgeBasesInTx(tx *gorm.DB) error {
+	logrus.Println("Initializing default knowledge bases in transaction...")
+
+	var adminUser model.User
+	if err := tx.Where("username = ?", "admin").First(&adminUser).Error; err != nil {
+		return err
+	}
+
+	var count int64
+	tx.Model(&model.KnowledgeBase{}).Where("name = ?", "默认知识库").Count(&count)
+	if count > 0 {
+		logrus.Println("Default knowledge base already exists in transaction.")
+		return nil
+	}
+
+	defaultKnowledgeBase := model.KnowledgeBase{
+		ExternalID:    utils.GenerateExternalID(utils.ExternalIDKnowledgeBase),
+		Name:          "默认知识库",
+		Description:   "系统默认创建的知识库，用于存储常用文档和资料",
+		Cover:         "",
+		CreatorUserID: adminUser.ID,
+		IsPublic:      false,
+	}
+
+	if err := tx.Create(&defaultKnowledgeBase).Error; err != nil {
+		return err
+	}
+
+	exampleKnowledgeBase := model.KnowledgeBase{
+		ExternalID:    utils.GenerateExternalID(utils.ExternalIDKnowledgeBase),
+		Name:          "示例知识库",
+		Description:   "示例知识库，展示知识库功能",
+		Cover:         "",
+		CreatorUserID: adminUser.ID,
+		IsPublic:      true,
+	}
+
+	if err := tx.Create(&exampleKnowledgeBase).Error; err != nil {
+		return err
+	}
+
+	logrus.Println("Adding documents to default knowledge base...")
+
+	rootDocumentTitles := []string{
+		"项目概述",
+		"系统架构",
+		"开发规范",
+		"API文档",
+		"数据库设计",
+		"前端开发",
+		"后端开发",
+		"测试计划",
+		"部署指南",
+		"安全规范",
+		"用户手册",
+	}
+
+	for _, title := range rootDocumentTitles {
+		rootDoc, err := createDocumentWithAllTables(tx, adminUser.ID, DocumentInput{
+			Title:       title,
+			Content:     "# " + title + "\n\n这是" + title + "的详细文档内容。",
+			Summary:     title + "相关文档",
+			ParentID:    0,
+			Tags:        []string{title},
+			KnowledgeID: &defaultKnowledgeBase.ID,
+		})
+		if err != nil {
+			logrus.Warnf("Failed to create root document %s: %v", title, err)
+			continue
+		}
+
+		for j := 1; j <= 3; j++ {
+			childTitle := title + " - 子文档" + string(rune('0'+j))
+			childDoc, err := createDocumentWithAllTables(tx, adminUser.ID, DocumentInput{
+				Title:       childTitle,
+				Content:     "# " + childTitle + "\n\n这是" + childTitle + "的详细文档内容。",
+				Summary:     childTitle + "相关文档",
+				ParentID:    rootDoc.ID,
+				Tags:        []string{title, "子文档"},
+				KnowledgeID: &defaultKnowledgeBase.ID,
+			})
+			if err != nil {
+				logrus.Warnf("Failed to create child document %s: %v", childTitle, err)
+				continue
+			}
+
+			for k := 1; k <= 2; k++ {
+				grandchildTitle := childTitle + " - 孙文档" + string(rune('0'+k))
+				_, err := createDocumentWithAllTables(tx, adminUser.ID, DocumentInput{
+					Title:       grandchildTitle,
+					Content:     "# " + grandchildTitle + "\n\n这是" + grandchildTitle + "的详细文档内容。",
+					Summary:     grandchildTitle + "相关文档",
+					ParentID:    childDoc.ID,
+					Tags:        []string{title, "子文档", "孙文档"},
+					KnowledgeID: &defaultKnowledgeBase.ID,
+				})
+				if err != nil {
+					logrus.Warnf("Failed to create grandchild document %s: %v", grandchildTitle, err)
+				}
+			}
+		}
+	}
+
+	logrus.Println("Documents added to default knowledge base successfully")
+
+	logrus.Println("Default knowledge bases created successfully in transaction")
+	return nil
+}

cmd/db_init/main.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/constant"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		InitPostgres().
+		InitConsul().
+		RequireConsulEnabled()
+	if err := initializer.Err(); err != nil {
+		logrus.Fatalf("Init db_init failed: %v", err)
+	}
+	defer initializer.Shutdown()
+
+	if isDatabaseInitialized() {
+		logrus.Println("Database already initialized, skipping initialization steps")
+	} else {
+		if err := initializeDatabaseInTransaction(); err != nil {
+			logrus.Fatalf("Database initialization failed: %v", err)
+		}
+		if err := initializeConfigInConsul(context.Background()); err != nil {
+			logrus.Fatalf("Consul config initialization failed: %v", err)
+		}
+		if err := markDatabaseInitializedInConsul(context.Background()); err != nil {
+			logrus.Fatalf("Mark database initialized failed: %v", err)
+		}
+		logrus.Println("Database initialized successfully")
+	}
+
+	logrus.Println("All initialization tasks completed successfully!")
+}
+
+func isDatabaseInitialized() bool {
+	value, ok, err := storage.Consul.Get(context.Background(), utils.GenConfigKey(
+		constant.ConfigKey_First_System,
+		constant.ConfigKey_Second_Database,
+		constant.ConfigKey_Third_Initialized,
+	))
+	if err != nil || !ok {
+		return false
+	}
+	return value == constant.Database_Initialized_True
+}
+
+func initializeDatabaseInTransaction() error {
+	return utils.WithTransaction(storage.DB, func(tx *gorm.DB) error {
+		if err := initializePermissionsInTx(tx); err != nil {
+			return err
+		}
+
+		if err := createAdminUserInTx(tx); err != nil {
+			return err
+		}
+
+		if err := createTestUserIntx(tx); err != nil {
+			return err
+		}
+
+		if err := initializeDocumentsInTx(tx); err != nil {
+			return err
+		}
+
+		if err := initializeKnowledgeBasesInTx(tx); err != nil {
+			return err
+		}
+
+		if err := initializeTemplatesInTx(tx); err != nil {
+			return err
+		}
+
+		if err := createNormalUserInTx(tx); err != nil {
+			return err
+		}
+
+		if err := initializeUserGroupsInTx(tx); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}

cmd/db_init/other_user.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"strconv"
+
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+func createTestUserIntx(tx *gorm.DB) error {
+	logrus.Println("Creating test user in transaction...")
+	return createUserWithUsername(tx, "test")
+}
+
+func createNormalUserInTx(tx *gorm.DB) error {
+	logrus.Println("Creating 100 users in transaction...")
+
+	for i := 1; i <= 20; i++ {
+		username := "user" + strconv.Itoa(i)
+		if err := createUserWithUsername(tx, username); err != nil {
+			return err
+		}
+
+		var user model.User
+		if err := tx.Where("username = ?", username).First(&user).Error; err != nil {
+			return err
+		}
+		userNum := strconv.Itoa(i)
+		logrus.Printf("User%s created successfully with ID: %s in transaction.\n", userNum, user.ExternalID)
+
+		logrus.Printf("Creating knowledge base for user%s...\n", userNum)
+
+		kbExternalID := utils.GenerateExternalID(utils.ExternalIDKnowledgeBase)
+		knowledgeBase := model.KnowledgeBase{
+			ExternalID:    kbExternalID,
+			Name:          "User" + userNum + "'s Knowledge Base",
+			Description:   "Knowledge base created by user" + userNum,
+			CreatorUserID: user.ID,
+			IsPublic:      false,
+		}
+
+		var kbCount int64
+		tx.Model(&model.KnowledgeBase{}).Where("name = ?", knowledgeBase.Name).Count(&kbCount)
+		if kbCount == 0 {
+			if err := tx.Create(&knowledgeBase).Error; err != nil {
+				return err
+			}
+			logrus.Printf("Knowledge base created successfully for user%s: %s\n", userNum, knowledgeBase.Name)
+		} else {
+			logrus.Printf("Knowledge base for user%s already exists.\n", userNum)
+		}
+	}
+
+	return nil
+}

cmd/db_init/permissions.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"gorm.io/gorm"
+)
+
+func initializePermissionsInTx(tx *gorm.DB) error {
+	// logrus.Println("Starting permission initialization in transaction...")
+
+	// adminSubject := model.Subject{
+	// 	ID:   "1",
+	// 	Type: model.SubjectTypeUser,
+	// }
+
+	// if err := tx.Where("id = ?", adminSubject.ID).FirstOrCreate(&adminSubject).Error; err != nil {
+	// 	return err
+	// }
+	// logrus.Println("Admin subject created successfully in transaction.")
+
+	// logrus.Println("Permission initialization completed successfully in transaction.")
+	return nil
+}

cmd/db_init/template.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+type templateSeed struct {
+	Name            string
+	Description     string
+	Content         string
+	Scope           string
+	KnowledgeBaseID *int64
+	Tags            []string
+}
+
+func initializeTemplatesInTx(tx *gorm.DB) error {
+	logrus.Println("Initializing default templates in transaction...")
+
+	var adminUser model.User
+	if err := tx.Where("username = ?", "admin").First(&adminUser).Error; err != nil {
+		return err
+	}
+
+	var defaultKB model.KnowledgeBase
+	if err := tx.Where("name = ?", "默认知识库").First(&defaultKB).Error; err != nil {
+		return err
+	}
+
+	seeds := []templateSeed{
+		{
+			Name:        "个人周报模板",
+			Description: "用于个人每周工作复盘与计划",
+			Content: `# 个人周报
+
+## 本周完成
+- 
+
+## 问题与风险
+- 
+
+## 下周计划
+- `,
+			Scope: "private",
+			Tags:  []string{"个人", "周报"},
+		},
+		{
+			Name:        "会议纪要模板",
+			Description: "通用会议记录模板",
+			Content: `# 会议纪要
+
+## 会议信息
+- 时间：
+- 参与人：
+
+## 讨论内容
+- 
+
+## 行动项
+- [ ] `,
+			Scope: "private",
+			Tags:  []string{"个人", "会议"},
+		},
+		{
+			Name:            "需求评审模板",
+			Description:     "知识库内需求评审记录模板",
+			KnowledgeBaseID: &defaultKB.ID,
+			Scope:           "knowledge_base",
+			Content: `# 需求评审
+
+## 背景
+
+## 目标
+
+## 方案评审
+- 优点：
+- 风险：
+
+## 结论`,
+			Tags: []string{"知识库", "需求"},
+		},
+		{
+			Name:            "技术方案模板",
+			Description:     "知识库内技术方案沉淀模板",
+			KnowledgeBaseID: &defaultKB.ID,
+			Scope:           "knowledge_base",
+			Content: `# 技术方案
+
+## 问题定义
+
+## 方案设计
+
+## 数据模型
+
+## 发布计划
+`,
+			Tags: []string{"知识库", "技术"},
+		},
+	}
+
+	for _, seed := range seeds {
+		if err := createTemplateIfNotExists(tx, adminUser.ID, seed); err != nil {
+			return err
+		}
+	}
+
+	logrus.Println("Default templates initialized successfully in transaction")
+	return nil
+}
+
+func createTemplateIfNotExists(tx *gorm.DB, userID int64, seed templateSeed) error {
+	query := tx.Model(&model.Template{}).
+		Where("user_id = ? AND name = ? AND scope = ?", userID, seed.Name, seed.Scope)
+	if seed.KnowledgeBaseID == nil {
+		query = query.Where("knowledge_base_id IS NULL")
+	} else {
+		query = query.Where("knowledge_base_id = ?", *seed.KnowledgeBaseID)
+	}
+
+	var count int64
+	if err := query.Count(&count).Error; err != nil {
+		return err
+	}
+	if count > 0 {
+		return nil
+	}
+
+	template := model.Template{
+		Name:            seed.Name,
+		Description:     seed.Description,
+		DocumentType:    model.DocumentType_Markdown,
+		Content:         seed.Content,
+		UserID:          userID,
+		Scope:           seed.Scope,
+		KnowledgeBaseID: seed.KnowledgeBaseID,
+		IsPublic:        false,
+		Tags:            seed.Tags,
+	}
+	return tx.Create(&template).Error
+}

cmd/db_init/user_group.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+type seedUserGroup struct {
+	Name        string
+	Description string
+	MemberUsers []string
+}
+
+func initializeUserGroupsInTx(tx *gorm.DB) error {
+	logrus.Println("Creating user groups in transaction...")
+
+	adminUser, err := getUserByUsername(tx, "admin")
+	if err != nil {
+		return err
+	}
+	user2, err := getUserByUsername(tx, "user2")
+	if err != nil {
+		return err
+	}
+
+	seedGroups := []seedUserGroup{
+		{
+			Name:        "Admin Group",
+			Description: "Group for admin only",
+			MemberUsers: []string{"admin"},
+		},
+		{
+			Name:        "User2 Group",
+			Description: "Group for user2 only",
+			MemberUsers: []string{"user2"},
+		},
+		{
+			Name:        "Admin & User2 Group",
+			Description: "Group for admin and user2",
+			MemberUsers: []string{"admin", "user2"},
+		},
+		{
+			Name:        "Empty Group",
+			Description: "Group with no members",
+			MemberUsers: []string{},
+		},
+	}
+
+	for _, seed := range seedGroups {
+		group, err := findOrCreateGroup(tx, seed.Name, seed.Description, adminUser.ID)
+		if err != nil {
+			return err
+		}
+		for _, username := range seed.MemberUsers {
+			var userID int64
+			switch username {
+			case "admin":
+				userID = adminUser.ID
+			case "user2":
+				userID = user2.ID
+			default:
+				continue
+			}
+			if err := ensureMembership(tx, group.ID, userID); err != nil {
+				return err
+			}
+		}
+	}
+
+	logrus.Println("User groups initialized successfully in transaction.")
+	return nil
+}
+
+func getUserByUsername(tx *gorm.DB, username string) (*model.User, error) {
+	var user model.User
+	if err := tx.Where("username = ?", username).First(&user).Error; err != nil {
+		return nil, err
+	}
+	return &user, nil
+}
+
+func findOrCreateGroup(tx *gorm.DB, name, description string, creatorID int64) (*model.UserGroupMeta, error) {
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return nil, gorm.ErrInvalidValue
+	}
+
+	var group model.UserGroupMeta
+	if err := tx.Where("name = ?", name).First(&group).Error; err == nil {
+		return &group, nil
+	} else if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return nil, err
+	}
+
+	group = model.UserGroupMeta{
+		ExternalID:  utils.GenerateExternalID(utils.ExternalIDContextUserGroup),
+		Name:        name,
+		Description: strings.TrimSpace(description),
+		CreatorID:   creatorID,
+	}
+	if err := tx.Create(&group).Error; err != nil {
+		return nil, err
+	}
+	return &group, nil
+}
+
+func ensureMembership(tx *gorm.DB, groupID, userID int64) error {
+	var count int64
+	if err := tx.Model(&model.MembershipRelation{}).
+		Where("type = ? AND membership_id = ? AND user_id = ?", model.MembershipType_UserGroup, groupID, userID).
+		Count(&count).Error; err != nil {
+		return err
+	}
+	if count > 0 {
+		return nil
+	}
+	return tx.Create(&model.MembershipRelation{
+		Type:         model.MembershipType_UserGroup,
+		MembershipID: groupID,
+		UserID:       userID,
+	}).Error
+}

cmd/es_init/main.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"context"
+	"errors"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/config"
+	"github.com/XingfenD/yoresee_doc/internal/constant"
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/internal/search"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+const reindexBatchSize = 200
+
+var errESInitDisabled = errors.New("es_init disabled")
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		Check("check elasticsearch enabled", func() error {
+			if config.GlobalConfig == nil || !config.GlobalConfig.Elasticsearch.Enabled {
+				return errESInitDisabled
+			}
+			return nil
+		}).
+		InitPostgres().
+		InitConsul().
+		RequireConsulEnabled().
+		InitElasticsearch()
+	if err := initializer.Err(); err != nil {
+		if errors.Is(err, errESInitDisabled) {
+			logrus.Println("Elasticsearch disabled, skip es_init")
+			return
+		}
+		logrus.Fatalf("Init es_init failed: %v", err)
+	}
+
+	defer initializer.Shutdown()
+
+	ctx := context.Background()
+	indexName := search.DocumentIndexName()
+	initialized := isESInitialized(ctx)
+
+	exists, err := storage.ES.IndexExists(ctx, indexName)
+	if err != nil {
+		logrus.Fatalf("Check Elasticsearch index failed, index=%s, err=%v", indexName, err)
+	}
+
+	if initialized && exists {
+		logrus.Printf("Elasticsearch already initialized, index=%s", indexName)
+		return
+	}
+
+	if !exists {
+		if err := storage.ES.CreateIndex(ctx, indexName, search.BuildDocumentIndexMapping()); err != nil {
+			logrus.Fatalf("Create Elasticsearch index failed, index=%s, err=%v", indexName, err)
+		}
+		logrus.Printf("Elasticsearch index created, index=%s", indexName)
+	}
+
+	if err := reindexDocuments(ctx); err != nil {
+		logrus.Fatalf("Reindex documents failed: %v", err)
+	}
+
+	if err := markESInitializedInConsul(ctx); err != nil {
+		logrus.Fatalf("Mark Elasticsearch initialized failed: %v", err)
+	}
+
+	logrus.Println("Elasticsearch initialization completed successfully")
+}
+
+func isESInitialized(ctx context.Context) bool {
+	value, ok, err := storage.Consul.Get(ctx, utils.GenConfigKey(
+		constant.ConfigKey_First_System,
+		constant.ConfigKey_Second_ES,
+		constant.ConfigKey_Third_Initialized,
+	))
+	if err != nil || !ok {
+		return false
+	}
+	return value == constant.ES_Initialized_True
+}
+
+func markESInitializedInConsul(ctx context.Context) error {
+	return storage.Consul.Set(
+		ctx,
+		utils.GenConfigKey(
+			constant.ConfigKey_First_System,
+			constant.ConfigKey_Second_ES,
+			constant.ConfigKey_Third_Initialized,
+		),
+		constant.ES_Initialized_True,
+	)
+}
+
+func reindexDocuments(ctx context.Context) error {
+	logrus.Println("Start reindexing documents to Elasticsearch...")
+
+	var docs []model.Document
+	return storage.DB.
+		Model(&model.Document{}).
+		Where("deleted_at IS NULL").
+		FindInBatches(&docs, reindexBatchSize, func(tx *gorm.DB, batch int) error {
+			for i := range docs {
+				doc := docs[i]
+				if err := search.UpsertDocument(ctx, &doc); err != nil {
+					return err
+				}
+			}
+			logrus.Printf("Reindex batch done, batch=%d, size=%d", batch, len(docs))
+			return nil
+		}).Error
+}

cmd/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/config"
+	"github.com/XingfenD/yoresee_doc/internal/transport/connectserver"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		InitPostgres().
+		InitRedis().
+		InitConsul().
+		RequireConsulEnabled().
+		InitMinio().
+		InitElasticsearchAllowFail().
+		InitMQ().
+		InitRepository().
+		InitService()
+	if err := initializer.Err(); err != nil {
+		logrus.Fatalf("Init backend failed: %v", err)
+	}
+
+	grpcServer, grpcWebServer, err := connectserver.Start(
+		config.GlobalConfig.Server.GrpcPort,
+		config.GlobalConfig.Server.GrpcWebPort,
+		initializer.Repositories.DB,
+		initializer.Repositories.Redis,
+		initializer.Repositories,
+	)
+	if err != nil {
+		logrus.Fatalf("Start connect servers failed: %v", err)
+	}
+
+	waitForShutdown(initializer, grpcServer, grpcWebServer)
+}
+
+func waitForShutdown(initializer *bootstrap.Initializer, grpcServer, grpcWebServer *http.Server) {
+	signalCtx, stop := utils.ShutdownContext()
+	defer stop()
+
+	<-signalCtx.Done()
+	logrus.Info("Shutdown signal received, start draining")
+	connectserver.SetDraining(true)
+
+	shutdownServer("grpc-web", grpcWebServer, 10*time.Second)
+	shutdownServer("grpc", grpcServer, 10*time.Second)
+
+	initializer.Shutdown()
+}
+
+func shutdownServer(name string, server *http.Server, timeout time.Duration) {
+	if server == nil {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	if err := server.Shutdown(ctx); err != nil {
+		logrus.Errorf("Shutdown %s server failed: %v", name, err)
+	}
+}

cmd/migrate/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"github.com/sirupsen/logrus"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+)
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		InitPostgres()
+	if err := initializer.Err(); err != nil {
+		logrus.Fatalf("Init migrate failed: %v", err)
+	}
+	defer initializer.Shutdown()
+
+	logrus.Println("Starting migration...")
+
+	if err := runMigration(); err != nil {
+		logrus.Fatalf("Migration failed: %v", err)
+	}
+	logrus.Println("Database migration completed successfully")
+
+}

cmd/migrate/migration.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"github.com/XingfenD/yoresee_doc/internal/model"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+func runMigration() error {
+	if err := storage.DB.Exec("CREATE EXTENSION IF NOT EXISTS ltree").Error; err != nil {
+		return err
+	}
+	logrus.Println("ltree extension created successfully")
+
+	err := storage.DB.AutoMigrate(
+		&model.Attachment{},
+		&model.Document{},
+		&model.DocumentComment{},
+		&model.DocumentYjsSnapshot{},
+		&model.DocumentVersion{},
+		&model.Invitation{},
+		&model.InvitationRecord{},
+		&model.KnowledgeBase{},
+		&model.RecentDocument{},
+		&model.RecentKnowledgeBase{},
+		&model.RecentTemplate{},
+		&model.MembershipRelation{},
+		&model.Notification{},
+		&model.OrgNodeMeta{},
+		&model.Template{},
+		&model.UserGroupMeta{},
+		&model.User{},
+	)
+	if err != nil {
+		return err
+	}
+
+	if err := migrateRecentKnowledgeBaseIndex(storage.DB); err != nil {
+		return err
+	}
+	if err := migrateRecentDocumentIndex(storage.DB); err != nil {
+		return err
+	}
+	if err := migrateRecentTemplateIndex(storage.DB); err != nil {
+		return err
+	}
+	if err := dropDocumentCommentQuote(storage.DB); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func migrateRecentKnowledgeBaseIndex(db *gorm.DB) error {
+	return db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_recent_kb_user_kb ON recent_knowledge_bases (user_id, knowledge_base_id)").Error
+}
+
+func migrateRecentTemplateIndex(db *gorm.DB) error {
+	return db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_recent_tpl_user_tpl ON recent_templates (user_id, template_id)").Error
+}
+
+func migrateRecentDocumentIndex(db *gorm.DB) error {
+	return db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_recent_doc_user_doc ON recent_documents (user_id, document_id)").Error
+}
+
+func dropDocumentCommentQuote(db *gorm.DB) error {
+	return db.Exec("ALTER TABLE document_comments DROP COLUMN IF EXISTS quote").Error
+}

cmd/notification-worker/main.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/domain_event"
+	"github.com/XingfenD/yoresee_doc/internal/dto"
+	"github.com/XingfenD/yoresee_doc/internal/service/mq_service"
+	"github.com/XingfenD/yoresee_doc/internal/service/notification_service"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/mq"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		InitPostgres().
+		InitRedis().
+		InitMQ().
+		InitRepository().
+		InitService()
+	if err := initializer.Err(); err != nil {
+		logrus.Fatalf("Init notification-worker failed: %v", err)
+	}
+
+	topic := domain_event.NotificationCreateTopic()
+	group := utils.GetEnvVar("NOTIFICATION_MQ_GROUP", "notification-worker")
+
+	go func() {
+		if err := mq_service.MQSvc.Consume(
+			context.Background(),
+			mq.BackendRabbitMQ,
+			mq.ConsumeOptions{
+				Topic:   topic,
+				Mode:    mq.ConsumeModeGroup,
+				Group:   group,
+				AutoAck: false,
+				OnError: mq.ErrorActionRequeue,
+			},
+			func(ctx context.Context, message mq.Message) error {
+				return handleNotificationEvent(ctx, message.Body)
+			},
+		); err != nil {
+			logrus.Fatalf("Subscribe failed: %v", err)
+		}
+	}()
+
+	initializer.ShutdownOnSignal(500 * time.Millisecond)
+}
+
+func handleNotificationEvent(ctx context.Context, data []byte) error {
+	payload := strings.TrimSpace(string(data))
+	if payload == "" {
+		return nil
+	}
+
+	evt, err := domain_event.DecodeNotificationCreateEvent(data)
+	if err != nil {
+		logrus.Errorf("parse notification event failed: %v", err)
+		return nil
+	}
+
+	req := &dto.CreateNotificationRequest{
+		ReceiverExternalIDs: evt.ReceiverExternalIDs,
+		Type:                evt.Type,
+		Title:               evt.Title,
+		Content:             evt.Content,
+		PayloadJSON:         evt.PayloadJSON,
+	}
+	if err := notification_service.NotificationSvc.CreateNotifications(req); err != nil {
+		logrus.Errorf("create notifications failed: %v", err)
+		return err
+	}
+	return nil
+}

cmd/search-sync-worker/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/config"
+	"github.com/XingfenD/yoresee_doc/internal/domain_event"
+	"github.com/XingfenD/yoresee_doc/internal/repository/document_repo"
+	"github.com/XingfenD/yoresee_doc/internal/search"
+	"github.com/XingfenD/yoresee_doc/internal/service/mq_service"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/mq"
+	"github.com/sirupsen/logrus"
+)
+
+var errSearchSyncWorkerDisabled = errors.New("search-sync-worker disabled")
+
+func main() {
+	initializer, err := initSearchSyncWorker()
+	if err != nil {
+		if errors.Is(err, errSearchSyncWorkerDisabled) {
+			logrus.Println("Elasticsearch disabled, skip search-sync-worker")
+			return
+		}
+		logrus.Fatalf("Init search-sync-worker failed: %v", err)
+	}
+
+	backend := mq.BackendRabbitMQ
+	topic := domain_event.DocumentSyncTopic()
+	group := utils.GetEnvVar("SEARCH_SYNC_MQ_GROUP", "search-sync-worker")
+	logrus.Infof("Search sync worker started: backend=%s topic=%s group=%s", backend, topic, group)
+
+	go func() {
+		if err := mq_service.MQSvc.Consume(
+			context.Background(),
+			backend,
+			mq.ConsumeOptions{
+				Topic:   topic,
+				Mode:    mq.ConsumeModeGroup,
+				Group:   group,
+				AutoAck: false,
+				OnError: mq.ErrorActionRequeue,
+			},
+			func(ctx context.Context, message mq.Message) error {
+				return handleDocumentEvent(ctx, message.Body, initializer.Repositories.Document)
+			},
+		); err != nil {
+			logrus.Fatalf("Subscribe search sync topic failed: %v", err)
+		}
+	}()
+
+	initializer.ShutdownOnSignal(500 * time.Millisecond)
+}
+
+func initSearchSyncWorker() (*bootstrap.Initializer, error) {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		Check("check elasticsearch enabled", func() error {
+			if config.GlobalConfig == nil || !config.GlobalConfig.Elasticsearch.Enabled {
+				return errSearchSyncWorkerDisabled
+			}
+			return nil
+		}).
+		InitPostgres().
+		InitRedis().
+		InitElasticsearch().
+		InitMQ().
+		InitRepository()
+	return initializer, initializer.Err()
+}
+
+func handleDocumentEvent(ctx context.Context, data []byte, docRepo *document_repo.DocumentRepository) error {
+	evt, err := domain_event.DecodeDocumentSyncEvent(data)
+	if err != nil {
+		logrus.Warnf("Parse search sync event failed: %v", err)
+		return nil
+	}
+
+	switch evt.Action {
+	case domain_event.DocumentActionUpsert:
+		doc, err := docRepo.GetByExternalID(evt.ExternalID).Exec(ctx)
+		if err != nil {
+			logrus.Warnf("Search sync load document failed, external_id=%s, err=%v", evt.ExternalID, err)
+			return err
+		}
+		if err := search.UpsertDocument(ctx, doc); err != nil {
+			logrus.Warnf("Search sync upsert failed, external_id=%s, err=%v", evt.ExternalID, err)
+			return err
+		}
+		logrus.Infof("Search sync upsert success, external_id=%s", evt.ExternalID)
+	case domain_event.DocumentActionDelete:
+		// reserved for future hard-delete sync.
+	default:
+		logrus.Warnf("Search sync skip unknown action, action=%s, external_id=%s", evt.Action, evt.ExternalID)
+	}
+	return nil
+}

cmd/snapshot-worker/main.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/XingfenD/yoresee_doc/internal/bootstrap"
+	"github.com/XingfenD/yoresee_doc/internal/service/mq_service"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/constant"
+	"github.com/XingfenD/yoresee_doc/pkg/key"
+	"github.com/XingfenD/yoresee_doc/pkg/mq"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	initializer := bootstrap.NewInitializer().
+		InitConfig().
+		InitPostgres().
+		InitRedis().
+		InitElasticsearchAllowFail().
+		InitMQ().
+		InitRepository().
+		InitService()
+	if err := initializer.Err(); err != nil {
+		logrus.Fatalf("Init snapshot-worker failed: %v", err)
+	}
+
+	w := &Worker{
+		collabCoreHTTP: utils.GetEnvVar("COLLAB_CORE_HTTP", "http://collab-core:1234"),
+		mqGroup:        utils.GetEnvVar("SNAPSHOT_MQ_GROUP", "snapshot-worker"),
+		topic:          constant.DirtyDocTopicDefault,
+		mqBackend:      mq.BackendRabbitMQ,
+		client:         &http.Client{Timeout: 10 * time.Second},
+		dirtySetKey:    key.KeyCollabDirtyDocSet(),
+		inFlight:       &sync.Map{},
+	}
+
+	logrus.Infof("Snapshot worker started: topic=%s group=%s collabCore=%s", w.topic, w.mqGroup, w.collabCoreHTTP)
+
+	go w.runMQConsumer(mq_service.MQSvc)
+	go w.runScanLoop()
+
+	initializer.ShutdownOnSignal(0)
+}

cmd/snapshot-worker/message.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"strings"
+
+	"github.com/bytedance/sonic"
+)
+
+type dirtyDocMessage struct {
+	DocID string `json:"doc_id"`
+	DocId string `json:"docId"`
+}
+
+func parseDocID(data []byte) string {
+	payload := strings.TrimSpace(string(data))
+	if payload == "" {
+		return ""
+	}
+	var msg dirtyDocMessage
+	if err := sonic.Unmarshal(data, &msg); err == nil {
+		if msg.DocID != "" {
+			return msg.DocID
+		}
+		if msg.DocId != "" {
+			return msg.DocId
+		}
+	}
+	return payload
+}

cmd/snapshot-worker/snapshot.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/XingfenD/yoresee_doc/internal/domain_event"
+	"github.com/XingfenD/yoresee_doc/internal/service/document_service"
+	"github.com/XingfenD/yoresee_doc/internal/status"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/key"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+	"github.com/bytedance/sonic"
+	"github.com/sirupsen/logrus"
+)
+
+func (w *Worker) snapshotDoc(ctx context.Context, docID string, force bool) error {
+	if _, loaded := w.inFlight.LoadOrStore(docID, struct{}{}); loaded {
+		return nil
+	}
+	defer w.inFlight.Delete(docID)
+
+	logrus.Infof("Snapshot start docId=%s force=%v", docID, force)
+
+	docType, err := document_service.DocumentSvc.GetDocumentTypeByExternalID(ctx, docID)
+	if err != nil {
+		if errors.Is(err, status.StatusDocumentNotFound) {
+			logrus.Infof("Snapshot doc not found docId=%s, cleaning up Redis", docID)
+			cleanupCollabKeys(ctx, docID)
+			return nil
+		}
+		logrus.Errorf("Snapshot get doc type failed docId=%s err=%v", docID, err)
+		return err
+	}
+
+	state, content, err := w.fetchSnapshot(ctx, docID, string(docType))
+	if err != nil {
+		logrus.Errorf("Snapshot fetch failed docId=%s err=%v", docID, err)
+		return err
+	}
+	if len(state) == 0 {
+		logrus.Infof("Snapshot empty docId=%s", docID)
+		return nil
+	}
+	logrus.Infof("Snapshot fetched docId=%s bytes=%d", docID, len(state))
+
+	contentChanged, err := document_service.DocumentSvc.SaveDocumentSnapshotAndContent(ctx, docID, state, content)
+	if err != nil {
+		logrus.Errorf("Snapshot save failed docId=%s err=%v", docID, err)
+		return err
+	}
+	if contentChanged {
+		if pubErr := domain_event.PublishDocumentUpsertEvent(ctx, docID); pubErr != nil {
+			logrus.Warnf("Snapshot publish search sync event failed docId=%s err=%v", docID, pubErr)
+		}
+	}
+
+	if err := flushCollabKeys(ctx, docID, state); err != nil {
+		logrus.Errorf("Snapshot redis flush failed docId=%s err=%v", docID, err)
+		return err
+	}
+
+	if force {
+		logrus.Infof("Snapshot saved (mq) for %s", docID)
+	} else {
+		logrus.Infof("Snapshot saved (scan) for %s", docID)
+	}
+	return nil
+}
+
+func (w *Worker) fetchSnapshot(ctx context.Context, docID, docType string) (state []byte, content string, err error) {
+	url := fmt.Sprintf("%s/internal/yjs/doc-snapshot/%s?type=%s", strings.TrimRight(w.collabCoreHTTP, "/"), docID, docType)
+	resp, err := w.client.Get(url)
+	if err != nil {
+		return nil, "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, "", nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var payload struct {
+		State   string `json:"state"`
+		Content string `json:"content"`
+	}
+	if err := sonic.Unmarshal(body, &payload); err != nil {
+		return nil, "", err
+	}
+	if payload.State == "" {
+		return nil, payload.Content, nil
+	}
+
+	state, err = utils.DecodeBase64(payload.State)
+	if err != nil {
+		return nil, "", err
+	}
+	return state, payload.Content, nil
+}
+
+func flushCollabKeys(ctx context.Context, docID string, state []byte) error {
+	rdb := storage.GetRedis()
+	if rdb == nil {
+		return nil
+	}
+	updatesKey := key.KeyCollabDocUpdates(docID)
+	pipe := rdb.TxPipeline()
+	pipe.Del(ctx, updatesKey)
+	pipe.RPush(ctx, updatesKey, state)
+	pipe.SRem(ctx, key.KeyCollabDirtyDocSet(), docID)
+	_, err := pipe.Exec(ctx)
+	return err
+}
+
+func cleanupCollabKeys(ctx context.Context, docID string) {
+	rdb := storage.GetRedis()
+	if rdb == nil {
+		return
+	}
+	pipe := rdb.TxPipeline()
+	pipe.SRem(ctx, key.KeyCollabDirtyDocSet(), docID)
+	pipe.Del(ctx, key.KeyCollabDocUpdates(docID))
+	pipe.Del(ctx, key.KeyCollabRoom(docID))
+	if _, err := pipe.Exec(ctx); err != nil {
+		logrus.Warnf("Snapshot cleanup Redis failed docId=%s err=%v", docID, err)
+	}
+}

cmd/snapshot-worker/worker.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/XingfenD/yoresee_doc/internal/service/mq_service"
+	"github.com/XingfenD/yoresee_doc/internal/utils"
+	"github.com/XingfenD/yoresee_doc/pkg/key"
+	"github.com/XingfenD/yoresee_doc/pkg/mq"
+	"github.com/XingfenD/yoresee_doc/pkg/storage"
+	"github.com/sirupsen/logrus"
+)
+
+type Worker struct {
+	collabCoreHTTP string
+	mqGroup        string
+	topic          string
+	mqBackend      mq.Backend
+	client         *http.Client
+	dirtySetKey    string
+	inFlight       *sync.Map
+}
+
+func (w *Worker) runMQConsumer(mqSvc *mq_service.MQService) {
+	err := mqSvc.Consume(
+		context.Background(),
+		w.mqBackend,
+		mq.ConsumeOptions{
+			Topic:   w.topic,
+			Mode:    mq.ConsumeModeGroup,
+			Group:   w.mqGroup,
+			AutoAck: false,
+			OnError: mq.ErrorActionRequeue,
+		},
+		func(ctx context.Context, message mq.Message) error {
+			docID := parseDocID(message.Body)
+			if docID == "" {
+				return nil
+			}
+			return w.snapshotDoc(ctx, docID, true)
+		},
+	)
+	if err != nil {
+		logrus.Fatalf("MQ consumer failed: %v", err)
+	}
+}
+
+func (w *Worker) runScanLoop() {
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+	for range ticker.C {
+		w.scanAndSnapshot()
+	}
+}
+
+func (w *Worker) scanAndSnapshot() {
+	ctx := context.Background()
+	rdb := storage.GetRedis()
+	if rdb == nil {
+		return
+	}
+
+	docIDs, err := rdb.SMembers(ctx, w.dirtySetKey).Result()
+	if err != nil {
+		logrus.Errorf("Scan dirty docs failed: %v", err)
+		return
+	}
+
+	candidates := filterCandidates(ctx, docIDs)
+	logrus.Infof("Scan dirty docs: candidates=%d", len(candidates))
+	if len(candidates) > 0 {
+		logrus.Infof("Dirty doc candidates: %v", candidates)
+	}
+	for _, docID := range candidates {
+		_ = w.snapshotDoc(ctx, docID, false)
+	}
+}
+
+func filterCandidates(ctx context.Context, docIDs []string) []string {
+	now := time.Now().UnixMilli()
+	candidates := make([]string, 0, len(docIDs))
+	for _, docID := range docIDs {
+		if docID == "" {
+			continue
+		}
+		lastStr, err := storage.GetRedis().Get(ctx, key.KeyCollabRoom(docID)).Result()
+		if err != nil {
+			logrus.Infof("Dirty doc %s skipped: room key missing", docID)
+			continue
+		}
+		last, err := utils.ParseInt64(lastStr)
+		if err != nil {
+			logrus.Infof("Dirty doc %s skipped: invalid room timestamp %s", docID, lastStr)
+			continue
+		}
+		if now-last < 10_000 {
+			logrus.Infof("Dirty doc %s skipped: lastEditAgo=%dms", docID, now-last)
+			continue
+		}
+		candidates = append(candidates, docID)
+	}
+	return candidates
+}