feat: implement proto-gen platform

- Config loader with env var expansion and validation - SQLite database with tasks and toolchain_versions tables - Gitea REST API client for repository and branch operations - K8s client for Job lifecycle management (with noop fallback for local dev) - REST API: tasks CRUD, retry, plugins, version management - Worker/scheduler with polling, concurrency control, and job tracking - Static frontend: generate, tasks, settings pages - Toolchain images: Go and Node.js Dockerfiles with entrypoint scripts - Helm chart with deployment, service, configmap, ingress, PVC - Multi-stage Dockerfile for service - Unit tests for config, database, gitclient, k8s, api, worker - E2E tests for idempotency, version switch, failure recovery
2026-08-10 16:54:30 +08:00
parent cc478bdad4
commit 4e467e1136
43 changed files with 3179 additions and 1 deletions
@@ -0,0 +1,70 @@
+package database
+
+import (
+	"database/sql"
+	"fmt"
+
+	_ "modernc.org/sqlite"
+)
+
+type DB struct {
+	conn *sql.DB
+}
+
+func New(dbPath string) (*DB, error) {
+	conn, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
+	if err != nil {
+		return nil, fmt.Errorf("opening database: %w", err)
+	}
+
+	db := &DB{conn: conn}
+	if err := db.migrate(); err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("running migrations: %w", err)
+	}
+
+	return db, nil
+}
+
+func (db *DB) Close() error {
+	return db.conn.Close()
+}
+
+func (db *DB) migrate() error {
+	schema := `
+	CREATE TABLE IF NOT EXISTS tasks (
+		id TEXT PRIMARY KEY,
+		status TEXT NOT NULL DEFAULT 'pending',
+		language TEXT NOT NULL,
+		proto_repo TEXT NOT NULL,
+		proto_branch TEXT NOT NULL,
+		proto_commit TEXT NOT NULL DEFAULT '',
+		target_repo TEXT NOT NULL,
+		toolchain_image TEXT NOT NULL,
+		image_digest TEXT NOT NULL DEFAULT '',
+		result_commit TEXT NOT NULL DEFAULT '',
+		error TEXT NOT NULL DEFAULT '',
+		log TEXT NOT NULL DEFAULT '',
+		retries INTEGER NOT NULL DEFAULT 0,
+		created_at TEXT NOT NULL,
+		updated_at TEXT NOT NULL
+	);
+
+	CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_running_per_repo
+		ON tasks(target_repo) WHERE status = 'running';
+
+	CREATE TABLE IF NOT EXISTS toolchain_versions (
+		language TEXT PRIMARY KEY,
+		image_tag TEXT NOT NULL,
+		enabled INTEGER NOT NULL DEFAULT 1,
+		updated_at TEXT NOT NULL
+	);
+	`
+
+	_, err := db.conn.Exec(schema)
+	return err
+}
+
+func (db *DB) Conn() *sql.DB {
+	return db.conn
+}
@@ -0,0 +1,97 @@
+package database
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"proto-gen/internal/models"
+)
+
+func (db *DB) CreateTask(task *models.Task) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	_, err := db.conn.Exec(`
+		INSERT INTO tasks (id, status, language, proto_repo, proto_branch, target_repo, toolchain_image, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		task.ID, task.Status, task.Language, task.ProtoRepo, task.ProtoBranch,
+		task.TargetRepo, task.ToolchainImage, now, now,
+	)
+	return err
+}
+
+func (db *DB) GetTask(id string) (*models.Task, error) {
+	var t models.Task
+	var createdAt, updatedAt string
+	err := db.conn.QueryRow(`
+		SELECT id, status, language, proto_repo, proto_branch, proto_commit, target_repo,
+		       toolchain_image, image_digest, result_commit, error, log, retries, created_at, updated_at
+		FROM tasks WHERE id = ?`, id,
+	).Scan(&t.ID, &t.Status, &t.Language, &t.ProtoRepo, &t.ProtoBranch, &t.ProtoCommit,
+		&t.TargetRepo, &t.ToolchainImage, &t.ImageDigest, &t.ResultCommit,
+		&t.Error, &t.Log, &t.Retries, &createdAt, &updatedAt)
+	if err == sql.ErrNoRows {
+		return nil, fmt.Errorf("task not found: %s", id)
+	}
+	if err != nil {
+		return nil, err
+	}
+	t.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
+	t.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt)
+	return &t, nil
+}
+
+func (db *DB) UpdateTask(task *models.Task) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	_, err := db.conn.Exec(`
+		UPDATE tasks SET status=?, proto_commit=?, image_digest=?, result_commit=?,
+		       error=?, log=?, retries=?, updated_at=?
+		WHERE id=?`,
+		task.Status, task.ProtoCommit, task.ImageDigest, task.ResultCommit,
+		task.Error, task.Log, task.Retries, now, task.ID,
+	)
+	return err
+}
+
+func (db *DB) ListTasks(status string, limit, offset int) ([]*models.Task, error) {
+	query := `SELECT id, status, language, proto_repo, proto_branch, proto_commit, target_repo,
+	          toolchain_image, image_digest, result_commit, error, log, retries, created_at, updated_at
+	          FROM tasks`
+	var args []interface{}
+	if status != "" {
+		query += ` WHERE status = ?`
+		args = append(args, status)
+	}
+	query += ` ORDER BY created_at DESC LIMIT ? OFFSET ?`
+	args = append(args, limit, offset)
+
+	rows, err := db.conn.Query(query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var tasks []*models.Task
+	for rows.Next() {
+		var t models.Task
+		var createdAt, updatedAt string
+		if err := rows.Scan(&t.ID, &t.Status, &t.Language, &t.ProtoRepo, &t.ProtoBranch,
+			&t.ProtoCommit, &t.TargetRepo, &t.ToolchainImage, &t.ImageDigest,
+			&t.ResultCommit, &t.Error, &t.Log, &t.Retries, &createdAt, &updatedAt); err != nil {
+			return nil, err
+		}
+		t.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
+		t.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt)
+		tasks = append(tasks, &t)
+	}
+	return tasks, nil
+}
+
+func (db *DB) GetPendingTasks(limit int) ([]*models.Task, error) {
+	return db.ListTasks(string(models.StatusPending), limit, 0)
+}
+
+func (db *DB) HasRunningTask(targetRepo string) (bool, error) {
+	var count int
+	err := db.conn.QueryRow(`SELECT COUNT(*) FROM tasks WHERE target_repo = ? AND status = 'running'`, targetRepo).Scan(&count)
+	return count > 0, err
+}
@@ -0,0 +1,124 @@
+package database
+
+import (
+	"testing"
+
+	"proto-gen/internal/models"
+)
+
+func newTestDB(t *testing.T) *DB {
+	t.Helper()
+	db, err := New(t.TempDir() + "/test.db")
+	if err != nil {
+		t.Fatalf("creating test db: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+	return db
+}
+
+func TestNew_CreatesTables(t *testing.T) {
+	db := newTestDB(t)
+	var count int
+	err := db.conn.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('tasks', 'toolchain_versions')`).Scan(&count)
+	if err != nil {
+		t.Fatalf("querying tables: %v", err)
+	}
+	if count != 2 {
+		t.Errorf("expected 2 tables, got %d", count)
+	}
+}
+
+func TestCreateAndGetTask(t *testing.T) {
+	db := newTestDB(t)
+	task := &models.Task{
+		ID:             "test-1",
+		Status:         models.StatusPending,
+		Language:       "go",
+		ProtoRepo:      "yoresee_doc",
+		ProtoBranch:    "main",
+		TargetRepo:     "yoresee_doc-gen-go",
+		ToolchainImage: "proto-gen-go:v1.0",
+	}
+	if err := db.CreateTask(task); err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	got, err := db.GetTask("test-1")
+	if err != nil {
+		t.Fatalf("GetTask: %v", err)
+	}
+	if got.Language != "go" {
+		t.Errorf("Language = %q, want %q", got.Language, "go")
+	}
+	if got.Status != models.StatusPending {
+		t.Errorf("Status = %q, want %q", got.Status, models.StatusPending)
+	}
+}
+
+func TestHasRunningTask(t *testing.T) {
+	db := newTestDB(t)
+
+	has, err := db.HasRunningTask("yoresee_doc-gen-go")
+	if err != nil {
+		t.Fatalf("HasRunningTask: %v", err)
+	}
+	if has {
+		t.Error("expected no running task")
+	}
+
+	task := &models.Task{
+		ID:             "test-2",
+		Status:         models.StatusRunning,
+		Language:       "go",
+		ProtoRepo:      "yoresee_doc",
+		ProtoBranch:    "main",
+		TargetRepo:     "yoresee_doc-gen-go",
+		ToolchainImage: "proto-gen-go:v1.0",
+	}
+	if err := db.CreateTask(task); err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	has, err = db.HasRunningTask("yoresee_doc-gen-go")
+	if err != nil {
+		t.Fatalf("HasRunningTask: %v", err)
+	}
+	if !has {
+		t.Error("expected running task")
+	}
+}
+
+func TestListTasks(t *testing.T) {
+	db := newTestDB(t)
+	for i := 0; i < 3; i++ {
+		status := models.StatusPending
+		if i == 1 {
+			status = models.StatusRunning
+		}
+		db.CreateTask(&models.Task{
+			ID:             "t-" + string(rune('0'+i)),
+			Status:         status,
+			Language:       "go",
+			ProtoRepo:      "yoresee_doc",
+			ProtoBranch:    "main",
+			TargetRepo:     "repo-" + string(rune('0'+i)),
+			ToolchainImage: "img:v1",
+		})
+	}
+
+	tasks, err := db.ListTasks("", 10, 0)
+	if err != nil {
+		t.Fatalf("ListTasks: %v", err)
+	}
+	if len(tasks) != 3 {
+		t.Errorf("len(tasks) = %d, want 3", len(tasks))
+	}
+
+	tasks, err = db.ListTasks(string(models.StatusRunning), 10, 0)
+	if err != nil {
+		t.Fatalf("ListTasks filtered: %v", err)
+	}
+	if len(tasks) != 1 {
+		t.Errorf("len(running) = %d, want 1", len(tasks))
+	}
+}