feat: implement proto-gen platform

- Config loader with env var expansion and validation
- SQLite database with tasks and toolchain_versions tables
- Gitea REST API client for repository and branch operations
- K8s client for Job lifecycle management (with noop fallback for local dev)
- REST API: tasks CRUD, retry, plugins, version management
- Worker/scheduler with polling, concurrency control, and job tracking
- Static frontend: generate, tasks, settings pages
- Toolchain images: Go and Node.js Dockerfiles with entrypoint scripts
- Helm chart with deployment, service, configmap, ingress, PVC
- Multi-stage Dockerfile for service
- Unit tests for config, database, gitclient, k8s, api, worker
- E2E tests for idempotency, version switch, failure recovery
This commit is contained in:
2026-08-10 16:54:30 +08:00
parent cc478bdad4
commit 4e467e1136
43 changed files with 3179 additions and 1 deletions
+145
View File
@@ -0,0 +1,145 @@
package worker
import (
"context"
"fmt"
"log"
"time"
"proto-gen/internal/config"
"proto-gen/internal/database"
"proto-gen/internal/k8s"
"proto-gen/internal/models"
)
type Scheduler struct {
db *database.DB
k8sClient k8s.K8sClient
cfg *config.Config
pollInterval time.Duration
running map[string]bool
}
func NewScheduler(db *database.DB, k8sClient k8s.K8sClient, cfg *config.Config) *Scheduler {
return &Scheduler{
db: db,
k8sClient: k8sClient,
cfg: cfg,
pollInterval: 5 * time.Second,
running: make(map[string]bool),
}
}
func (s *Scheduler) Start(ctx context.Context) {
log.Println("scheduler started")
ticker := time.NewTicker(s.pollInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
log.Println("scheduler stopped")
return
case <-ticker.C:
s.processPendingTasks()
s.trackRunningJobs()
}
}
}
func (s *Scheduler) processPendingTasks() {
pendingTasks, err := s.db.GetPendingTasks(10)
if err != nil {
log.Printf("error getting pending tasks: %v", err)
return
}
for _, task := range pendingTasks {
if len(s.running) >= s.cfg.Cluster.MaxConcurrent {
log.Printf("max concurrent reached, skipping task %s", task.ID)
return
}
hasRunning, err := s.db.HasRunningTask(task.TargetRepo)
if err != nil {
log.Printf("error checking running task: %v", err)
continue
}
if hasRunning {
log.Printf("task %s: target repo %s already has running task", task.ID, task.TargetRepo)
continue
}
s.startTask(task)
}
}
func (s *Scheduler) startTask(task *models.Task) {
envVars := map[string]string{
"GIT_BASE_URL": s.cfg.Git.BaseURL,
"GIT_TOKEN": s.cfg.Git.Token,
"PROTO_REPO": task.ProtoRepo,
"PROTO_BRANCH": task.ProtoBranch,
"TARGET_REPO": task.TargetRepo,
"OUT_DIR": "/out",
}
jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
job, err := s.k8sClient.CreateJob(&k8s.JobSpec{
Name: jobName,
Namespace: s.cfg.Cluster.Namespace,
Image: task.ToolchainImage,
EnvVars: envVars,
TimeoutSecond: s.cfg.Cluster.JobTimeout,
})
if err != nil {
log.Printf("task %s: failed to create job: %v", task.ID, err)
task.Status = models.StatusFailed
task.Error = fmt.Sprintf("failed to create K8s job: %v", err)
s.db.UpdateTask(task)
return
}
task.Status = models.StatusRunning
if err := s.db.UpdateTask(task); err != nil {
log.Printf("task %s: failed to update status: %v", task.ID, err)
return
}
s.running[task.ID] = true
log.Printf("task %s: job %s created", task.ID, job.Name)
}
func (s *Scheduler) trackRunningJobs() {
for taskID := range s.running {
task, err := s.db.GetTask(taskID)
if err != nil {
delete(s.running, taskID)
continue
}
jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
job, err := s.k8sClient.GetJob(jobName, s.cfg.Cluster.Namespace)
if err != nil {
log.Printf("task %s: error getting job: %v", taskID, err)
continue
}
switch job.Status {
case k8s.JobSucceeded:
task.Status = models.StatusSuccess
task.Log = job.Logs
s.db.UpdateTask(task)
delete(s.running, taskID)
log.Printf("task %s: completed successfully", taskID)
case k8s.JobFailed:
task.Status = models.StatusFailed
task.Error = "K8s job failed"
task.Log = job.Logs
s.db.UpdateTask(task)
delete(s.running, taskID)
log.Printf("task %s: failed", taskID)
}
}
}
+53
View File
@@ -0,0 +1,53 @@
package worker
import (
"testing"
"time"
"proto-gen/internal/config"
"proto-gen/internal/database"
"proto-gen/internal/k8s"
)
func newTestScheduler(t *testing.T) (*Scheduler, *database.DB) {
t.Helper()
db, err := database.New(t.TempDir() + "/test.db")
if err != nil {
t.Fatalf("creating db: %v", err)
}
t.Cleanup(func() { db.Close() })
cfg := &config.Config{
Cluster: config.ClusterConfig{
Namespace: "test",
JobTimeout: 300,
MaxConcurrent: 2,
},
ToolchainRegistry: "registry.example.com",
}
k8sClient := k8s.NewClientFromInterface(nil)
scheduler := NewScheduler(db, k8sClient, cfg)
scheduler.pollInterval = 100 * time.Millisecond
return scheduler, db
}
func TestNewScheduler(t *testing.T) {
scheduler, db := newTestScheduler(t)
defer db.Close()
if scheduler.db == nil {
t.Error("db is nil")
}
if scheduler.k8sClient == nil {
t.Error("k8sClient is nil")
}
}
func TestProcessPendingTasks_Empty(t *testing.T) {
scheduler, db := newTestScheduler(t)
defer db.Close()
scheduler.processPendingTasks()
}