feat: implement proto-gen platform

- Config loader with env var expansion and validation - SQLite database with tasks and toolchain_versions tables - Gitea REST API client for repository and branch operations - K8s client for Job lifecycle management (with noop fallback for local dev) - REST API: tasks CRUD, retry, plugins, version management - Worker/scheduler with polling, concurrency control, and job tracking - Static frontend: generate, tasks, settings pages - Toolchain images: Go and Node.js Dockerfiles with entrypoint scripts - Helm chart with deployment, service, configmap, ingress, PVC - Multi-stage Dockerfile for service - Unit tests for config, database, gitclient, k8s, api, worker - E2E tests for idempotency, version switch, failure recovery
2026-08-10 16:54:30 +08:00
parent cc478bdad4
commit 4e467e1136
43 changed files with 3179 additions and 1 deletions
@@ -0,0 +1,145 @@
+package worker
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"proto-gen/internal/config"
+	"proto-gen/internal/database"
+	"proto-gen/internal/k8s"
+	"proto-gen/internal/models"
+)
+
+type Scheduler struct {
+	db            *database.DB
+	k8sClient     k8s.K8sClient
+	cfg           *config.Config
+	pollInterval  time.Duration
+	running       map[string]bool
+}
+
+func NewScheduler(db *database.DB, k8sClient k8s.K8sClient, cfg *config.Config) *Scheduler {
+	return &Scheduler{
+		db:           db,
+		k8sClient:    k8sClient,
+		cfg:          cfg,
+		pollInterval: 5 * time.Second,
+		running:      make(map[string]bool),
+	}
+}
+
+func (s *Scheduler) Start(ctx context.Context) {
+	log.Println("scheduler started")
+	ticker := time.NewTicker(s.pollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Println("scheduler stopped")
+			return
+		case <-ticker.C:
+			s.processPendingTasks()
+			s.trackRunningJobs()
+		}
+	}
+}
+
+func (s *Scheduler) processPendingTasks() {
+	pendingTasks, err := s.db.GetPendingTasks(10)
+	if err != nil {
+		log.Printf("error getting pending tasks: %v", err)
+		return
+	}
+
+	for _, task := range pendingTasks {
+		if len(s.running) >= s.cfg.Cluster.MaxConcurrent {
+			log.Printf("max concurrent reached, skipping task %s", task.ID)
+			return
+		}
+
+		hasRunning, err := s.db.HasRunningTask(task.TargetRepo)
+		if err != nil {
+			log.Printf("error checking running task: %v", err)
+			continue
+		}
+		if hasRunning {
+			log.Printf("task %s: target repo %s already has running task", task.ID, task.TargetRepo)
+			continue
+		}
+
+		s.startTask(task)
+	}
+}
+
+func (s *Scheduler) startTask(task *models.Task) {
+	envVars := map[string]string{
+		"GIT_BASE_URL":   s.cfg.Git.BaseURL,
+		"GIT_TOKEN":      s.cfg.Git.Token,
+		"PROTO_REPO":     task.ProtoRepo,
+		"PROTO_BRANCH":   task.ProtoBranch,
+		"TARGET_REPO":    task.TargetRepo,
+		"OUT_DIR":        "/out",
+	}
+
+	jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
+	job, err := s.k8sClient.CreateJob(&k8s.JobSpec{
+		Name:          jobName,
+		Namespace:     s.cfg.Cluster.Namespace,
+		Image:         task.ToolchainImage,
+		EnvVars:       envVars,
+		TimeoutSecond: s.cfg.Cluster.JobTimeout,
+	})
+	if err != nil {
+		log.Printf("task %s: failed to create job: %v", task.ID, err)
+		task.Status = models.StatusFailed
+		task.Error = fmt.Sprintf("failed to create K8s job: %v", err)
+		s.db.UpdateTask(task)
+		return
+	}
+
+	task.Status = models.StatusRunning
+	if err := s.db.UpdateTask(task); err != nil {
+		log.Printf("task %s: failed to update status: %v", task.ID, err)
+		return
+	}
+
+	s.running[task.ID] = true
+	log.Printf("task %s: job %s created", task.ID, job.Name)
+}
+
+func (s *Scheduler) trackRunningJobs() {
+	for taskID := range s.running {
+		task, err := s.db.GetTask(taskID)
+		if err != nil {
+			delete(s.running, taskID)
+			continue
+		}
+
+		jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
+		job, err := s.k8sClient.GetJob(jobName, s.cfg.Cluster.Namespace)
+		if err != nil {
+			log.Printf("task %s: error getting job: %v", taskID, err)
+			continue
+		}
+
+		switch job.Status {
+		case k8s.JobSucceeded:
+			task.Status = models.StatusSuccess
+			task.Log = job.Logs
+			s.db.UpdateTask(task)
+			delete(s.running, taskID)
+			log.Printf("task %s: completed successfully", taskID)
+
+		case k8s.JobFailed:
+			task.Status = models.StatusFailed
+			task.Error = "K8s job failed"
+			task.Log = job.Logs
+			s.db.UpdateTask(task)
+			delete(s.running, taskID)
+			log.Printf("task %s: failed", taskID)
+		}
+	}
+}