feat: implement proto-gen platform
- Config loader with env var expansion and validation - SQLite database with tasks and toolchain_versions tables - Gitea REST API client for repository and branch operations - K8s client for Job lifecycle management (with noop fallback for local dev) - REST API: tasks CRUD, retry, plugins, version management - Worker/scheduler with polling, concurrency control, and job tracking - Static frontend: generate, tasks, settings pages - Toolchain images: Go and Node.js Dockerfiles with entrypoint scripts - Helm chart with deployment, service, configmap, ingress, PVC - Multi-stage Dockerfile for service - Unit tests for config, database, gitclient, k8s, api, worker - E2E tests for idempotency, version switch, failure recovery
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"proto-gen/internal/config"
|
||||
"proto-gen/internal/models"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, msg string) {
|
||||
writeJSON(w, status, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
type CreateTaskRequest struct {
|
||||
Language string `json:"language"`
|
||||
ProtoRepo string `json:"proto_repo"`
|
||||
ProtoBranch string `json:"proto_branch"`
|
||||
TargetRepo string `json:"target_repo"`
|
||||
}
|
||||
|
||||
func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
var req CreateTaskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Language == "" || req.ProtoRepo == "" || req.ProtoBranch == "" {
|
||||
writeError(w, http.StatusBadRequest, "language, proto_repo, and proto_branch are required")
|
||||
return
|
||||
}
|
||||
|
||||
plugin := s.cfg.GetPluginByLanguage(req.Language)
|
||||
if plugin == nil {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported language: %s", req.Language))
|
||||
return
|
||||
}
|
||||
|
||||
if req.TargetRepo == "" {
|
||||
req.TargetRepo = strings.ReplaceAll(
|
||||
strings.ReplaceAll(s.cfg.TargetRepoTemplate, "{proto_repo}", req.ProtoRepo),
|
||||
"{lang}", req.Language,
|
||||
)
|
||||
}
|
||||
|
||||
branch, err := s.git.GetBranch(req.ProtoRepo, req.ProtoRepo, req.ProtoBranch)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("branch not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
hasRunning, err := s.db.HasRunningTask(req.TargetRepo)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal error")
|
||||
return
|
||||
}
|
||||
if hasRunning {
|
||||
writeError(w, http.StatusConflict, "target repo already has a running task")
|
||||
return
|
||||
}
|
||||
|
||||
task := &models.Task{
|
||||
ID: uuid.New().String(),
|
||||
Status: models.StatusPending,
|
||||
Language: req.Language,
|
||||
ProtoRepo: req.ProtoRepo,
|
||||
ProtoBranch: req.ProtoBranch,
|
||||
ProtoCommit: branch.Commit,
|
||||
TargetRepo: req.TargetRepo,
|
||||
ToolchainImage: s.cfg.ToolchainRegistry + "/" + plugin.Image,
|
||||
}
|
||||
|
||||
if err := s.db.CreateTask(task); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to create task")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, task)
|
||||
}
|
||||
|
||||
func (s *Server) handleListTasks(w http.ResponseWriter, r *http.Request) {
|
||||
status := r.URL.Query().Get("status")
|
||||
limit := 50
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
offset := 0
|
||||
if o := r.URL.Query().Get("offset"); o != "" {
|
||||
if parsed, err := strconv.Atoi(o); err == nil && parsed >= 0 {
|
||||
offset = parsed
|
||||
}
|
||||
}
|
||||
|
||||
tasks, err := s.db.ListTasks(status, limit, offset)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to list tasks")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, tasks)
|
||||
}
|
||||
|
||||
func (s *Server) handleGetTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
task, err := s.db.GetTask(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, "task not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
func (s *Server) handleRetryTask(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
task, err := s.db.GetTask(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, "task not found")
|
||||
return
|
||||
}
|
||||
|
||||
if task.Status != models.StatusFailed {
|
||||
writeError(w, http.StatusBadRequest, "can only retry failed tasks")
|
||||
return
|
||||
}
|
||||
|
||||
hasRunning, err := s.db.HasRunningTask(task.TargetRepo)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal error")
|
||||
return
|
||||
}
|
||||
if hasRunning {
|
||||
writeError(w, http.StatusConflict, "target repo already has a running task")
|
||||
return
|
||||
}
|
||||
|
||||
task.Status = models.StatusPending
|
||||
task.Error = ""
|
||||
task.Log = ""
|
||||
task.Retries++
|
||||
if err := s.db.UpdateTask(task); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to retry task")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
}
|
||||
|
||||
func (s *Server) handleListPlugins(w http.ResponseWriter, r *http.Request) {
|
||||
var plugins []config.PluginConfig
|
||||
for _, p := range s.cfg.Plugins {
|
||||
plugins = append(plugins, p)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, plugins)
|
||||
}
|
||||
|
||||
func (s *Server) handleGetPluginVersion(w http.ResponseWriter, r *http.Request) {
|
||||
lang := chi.URLParam(r, "lang")
|
||||
plugin := s.cfg.GetPluginByLanguage(lang)
|
||||
if plugin == nil {
|
||||
writeError(w, http.StatusNotFound, "plugin not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, plugin)
|
||||
}
|
||||
|
||||
func (s *Server) handleUpdatePluginVersion(w http.ResponseWriter, r *http.Request) {
|
||||
lang := chi.URLParam(r, "lang")
|
||||
plugin := s.cfg.GetPluginByLanguage(lang)
|
||||
if plugin == nil {
|
||||
writeError(w, http.StatusNotFound, "plugin not found")
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Image string `json:"image"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
if req.Image == "" {
|
||||
writeError(w, http.StatusBadRequest, "image is required")
|
||||
return
|
||||
}
|
||||
|
||||
plugin.Image = req.Image
|
||||
writeJSON(w, http.StatusOK, plugin)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"proto-gen/internal/config"
|
||||
"proto-gen/internal/database"
|
||||
"proto-gen/internal/gitclient"
|
||||
"proto-gen/internal/models"
|
||||
)
|
||||
|
||||
type mockGitClient struct{}
|
||||
|
||||
func (m *mockGitClient) ListRepositories(owner string) ([]gitclient.Repository, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockGitClient) CreateRepository(owner, name, desc string, private bool) (*gitclient.Repository, error) {
|
||||
return &gitclient.Repository{Name: name}, nil
|
||||
}
|
||||
func (m *mockGitClient) GetRepository(owner, name string) (*gitclient.Repository, error) {
|
||||
return &gitclient.Repository{Name: name}, nil
|
||||
}
|
||||
func (m *mockGitClient) ListBranches(owner, repo string) ([]gitclient.Branch, error) {
|
||||
return []gitclient.Branch{{Name: "main", Commit: "abc123"}}, nil
|
||||
}
|
||||
func (m *mockGitClient) GetBranch(owner, repo, branch string) (*gitclient.Branch, error) {
|
||||
return &gitclient.Branch{Name: branch, Commit: "abc123"}, nil
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T) *Server {
|
||||
t.Helper()
|
||||
db, err := database.New(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatalf("creating db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
cfg := &config.Config{
|
||||
ToolchainRegistry: "registry.example.com/proto-gen",
|
||||
TargetRepoTemplate: "{proto_repo}-gen-{lang}",
|
||||
Plugins: []config.PluginConfig{
|
||||
{Language: "go", Image: "proto-gen-go:v1.0"},
|
||||
{Language: "nodejs", Image: "proto-gen-ts:v1.0"},
|
||||
},
|
||||
}
|
||||
|
||||
return NewServer(db, cfg, &mockGitClient{})
|
||||
}
|
||||
|
||||
func TestHandleCreateTask(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
body, _ := json.Marshal(CreateTaskRequest{
|
||||
Language: "go",
|
||||
ProtoRepo: "test-owner/yoresee_doc",
|
||||
ProtoBranch: "main",
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/v1/tasks", bytes.NewReader(body))
|
||||
w := httptest.NewRecorder()
|
||||
srv.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Errorf("status = %d, want %d", w.Code, http.StatusCreated)
|
||||
}
|
||||
|
||||
var task models.Task
|
||||
json.NewDecoder(w.Body).Decode(&task)
|
||||
if task.Language != "go" {
|
||||
t.Errorf("Language = %q, want %q", task.Language, "go")
|
||||
}
|
||||
if task.Status != models.StatusPending {
|
||||
t.Errorf("Status = %q, want %q", task.Status, models.StatusPending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCreateTask_UnsupportedLanguage(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
body, _ := json.Marshal(CreateTaskRequest{
|
||||
Language: "python",
|
||||
ProtoRepo: "test-owner/yoresee_doc",
|
||||
ProtoBranch: "main",
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/v1/tasks", bytes.NewReader(body))
|
||||
w := httptest.NewRecorder()
|
||||
srv.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("status = %d, want %d", w.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListPlugins(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/v1/plugins", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d", w.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
var plugins []config.PluginConfig
|
||||
json.NewDecoder(w.Body).Decode(&plugins)
|
||||
if len(plugins) != 2 {
|
||||
t.Errorf("len(plugins) = %d, want 2", len(plugins))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetTask_NotFound(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/v1/tasks/nonexistent", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"proto-gen/internal/config"
|
||||
"proto-gen/internal/database"
|
||||
"proto-gen/internal/gitclient"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
router *chi.Mux
|
||||
db *database.DB
|
||||
cfg *config.Config
|
||||
git gitclient.GitClient
|
||||
}
|
||||
|
||||
func NewServer(db *database.DB, cfg *config.Config, git gitclient.GitClient) *Server {
|
||||
s := &Server{
|
||||
router: chi.NewRouter(),
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
git: git,
|
||||
}
|
||||
s.routes()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) routes() {
|
||||
s.router.Use(middleware.Logger)
|
||||
s.router.Use(middleware.Recoverer)
|
||||
s.router.Use(middleware.Compress(5))
|
||||
|
||||
s.router.Route("/api/v1", func(r chi.Router) {
|
||||
r.Post("/tasks", s.handleCreateTask)
|
||||
r.Get("/tasks", s.handleListTasks)
|
||||
r.Get("/tasks/{id}", s.handleGetTask)
|
||||
r.Post("/tasks/{id}/retry", s.handleRetryTask)
|
||||
r.Get("/plugins", s.handleListPlugins)
|
||||
r.Get("/plugins/{lang}/versions", s.handleGetPluginVersion)
|
||||
r.Put("/plugins/{lang}/versions", s.handleUpdatePluginVersion)
|
||||
})
|
||||
|
||||
s.router.Handle("/*", http.FileServer(http.Dir("internal/static")))
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (s *Server) Start(addr string) error {
|
||||
log.Printf("server listening on %s", addr)
|
||||
return http.ListenAndServe(addr, s.router)
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Git GitConfig `yaml:"git"`
|
||||
Cluster ClusterConfig `yaml:"cluster"`
|
||||
ToolchainRegistry string `yaml:"toolchain_registry"`
|
||||
TargetRepoTemplate string `yaml:"target_repo_template"`
|
||||
Plugins []PluginConfig `yaml:"plugins"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
Addr string `yaml:"addr"`
|
||||
PublicURL string `yaml:"public_url"`
|
||||
}
|
||||
|
||||
type GitConfig struct {
|
||||
Type string `yaml:"type"`
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Token string `yaml:"token"`
|
||||
}
|
||||
|
||||
type ClusterConfig struct {
|
||||
Kubeconfig string `yaml:"kubeconfig"`
|
||||
Namespace string `yaml:"namespace"`
|
||||
JobTimeout int `yaml:"job_timeout"`
|
||||
MaxConcurrent int `yaml:"max_concurrent"`
|
||||
}
|
||||
|
||||
type PluginConfig struct {
|
||||
Language string `yaml:"language"`
|
||||
Image string `yaml:"image"`
|
||||
}
|
||||
|
||||
var envVarRegex = regexp.MustCompile(`\$\{(\w+)\}`)
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading config file: %w", err)
|
||||
}
|
||||
|
||||
expanded := expandEnvVars(string(data))
|
||||
|
||||
var cfg Config
|
||||
if err := yaml.Unmarshal([]byte(expanded), &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parsing config: %w", err)
|
||||
}
|
||||
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func expandEnvVars(s string) string {
|
||||
return envVarRegex.ReplaceAllStringFunc(s, func(match string) string {
|
||||
varName := match[2 : len(match)-1]
|
||||
if val, ok := os.LookupEnv(varName); ok {
|
||||
return val
|
||||
}
|
||||
return match
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Config) validate() error {
|
||||
if c.Git.Type == "" {
|
||||
return fmt.Errorf("git.type is required")
|
||||
}
|
||||
if c.Git.BaseURL == "" {
|
||||
return fmt.Errorf("git.base_url is required")
|
||||
}
|
||||
if c.Git.Token == "" {
|
||||
return fmt.Errorf("git.token is required")
|
||||
}
|
||||
if c.Cluster.Namespace == "" {
|
||||
return fmt.Errorf("cluster.namespace is required")
|
||||
}
|
||||
if c.Cluster.JobTimeout == 0 {
|
||||
c.Cluster.JobTimeout = 600
|
||||
}
|
||||
if c.Cluster.MaxConcurrent == 0 {
|
||||
c.Cluster.MaxConcurrent = 2
|
||||
}
|
||||
if c.Server.Addr == "" {
|
||||
c.Server.Addr = ":8080"
|
||||
}
|
||||
if c.TargetRepoTemplate == "" {
|
||||
c.TargetRepoTemplate = "{proto_repo}-gen-{lang}"
|
||||
}
|
||||
for i, p := range c.Plugins {
|
||||
if p.Language == "" {
|
||||
return fmt.Errorf("plugins[%d].language is required", i)
|
||||
}
|
||||
if p.Image == "" {
|
||||
return fmt.Errorf("plugins[%d].image is required", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Config) GetPluginByLanguage(lang string) *PluginConfig {
|
||||
for i := range c.Plugins {
|
||||
if strings.EqualFold(c.Plugins[i].Language, lang) {
|
||||
return &c.Plugins[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoad_MissingFile(t *testing.T) {
|
||||
_, err := Load("/nonexistent/config.yaml")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_MissingGitToken(t *testing.T) {
|
||||
cfg := `server:
|
||||
addr: ":8080"
|
||||
git:
|
||||
type: gitea
|
||||
base_url: "https://git.example.com"
|
||||
cluster:
|
||||
namespace: proto-gen`
|
||||
path := writeTempConfig(t, cfg)
|
||||
os.Setenv("GIT_TOKEN", "")
|
||||
_, err := Load(path)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing git.token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_ValidConfig(t *testing.T) {
|
||||
os.Setenv("GIT_TOKEN", "test-token")
|
||||
defer os.Unsetenv("GIT_TOKEN")
|
||||
|
||||
cfg := `server:
|
||||
addr: ":9090"
|
||||
git:
|
||||
type: gitea
|
||||
base_url: "https://git.example.com"
|
||||
token: ${GIT_TOKEN}
|
||||
cluster:
|
||||
namespace: proto-gen
|
||||
job_timeout: 300
|
||||
max_concurrent: 3
|
||||
plugins:
|
||||
- language: go
|
||||
image: "proto-gen-go:v1.0"`
|
||||
path := writeTempConfig(t, cfg)
|
||||
|
||||
got, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got.Server.Addr != ":9090" {
|
||||
t.Errorf("Server.Addr = %q, want %q", got.Server.Addr, ":9090")
|
||||
}
|
||||
if got.Git.Token != "test-token" {
|
||||
t.Errorf("Git.Token = %q, want %q", got.Git.Token, "test-token")
|
||||
}
|
||||
if got.Cluster.JobTimeout != 300 {
|
||||
t.Errorf("Cluster.JobTimeout = %d, want 300", got.Cluster.JobTimeout)
|
||||
}
|
||||
if len(got.Plugins) != 1 {
|
||||
t.Fatalf("len(Plugins) = %d, want 1", len(got.Plugins))
|
||||
}
|
||||
if got.Plugins[0].Language != "go" {
|
||||
t.Errorf("Plugins[0].Language = %q, want %q", got.Plugins[0].Language, "go")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_DefaultValues(t *testing.T) {
|
||||
os.Setenv("GIT_TOKEN", "test-token")
|
||||
defer os.Unsetenv("GIT_TOKEN")
|
||||
|
||||
cfg := `git:
|
||||
type: gitea
|
||||
base_url: "https://git.example.com"
|
||||
token: ${GIT_TOKEN}
|
||||
cluster:
|
||||
namespace: proto-gen`
|
||||
path := writeTempConfig(t, cfg)
|
||||
|
||||
got, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got.Server.Addr != ":8080" {
|
||||
t.Errorf("Server.Addr = %q, want %q", got.Server.Addr, ":8080")
|
||||
}
|
||||
if got.Cluster.JobTimeout != 600 {
|
||||
t.Errorf("Cluster.JobTimeout = %d, want 600", got.Cluster.JobTimeout)
|
||||
}
|
||||
if got.Cluster.MaxConcurrent != 2 {
|
||||
t.Errorf("Cluster.MaxConcurrent = %d, want 2", got.Cluster.MaxConcurrent)
|
||||
}
|
||||
if got.TargetRepoTemplate != "{proto_repo}-gen-{lang}" {
|
||||
t.Errorf("TargetRepoTemplate = %q, want %q", got.TargetRepoTemplate, "{proto_repo}-gen-{lang}")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPluginByLanguage(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Plugins: []PluginConfig{
|
||||
{Language: "go", Image: "go:v1"},
|
||||
{Language: "nodejs", Image: "ts:v1"},
|
||||
},
|
||||
}
|
||||
if p := cfg.GetPluginByLanguage("go"); p == nil {
|
||||
t.Error("GetPluginByLanguage(go) = nil")
|
||||
} else if p.Image != "go:v1" {
|
||||
t.Errorf("Image = %q, want %q", p.Image, "go:v1")
|
||||
}
|
||||
if p := cfg.GetPluginByLanguage("python"); p != nil {
|
||||
t.Error("GetPluginByLanguage(python) != nil")
|
||||
}
|
||||
}
|
||||
|
||||
func writeTempConfig(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("writing temp config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
}
|
||||
|
||||
func New(dbPath string) (*DB, error) {
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening database: %w", err)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn}
|
||||
if err := db.migrate(); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("running migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() error {
|
||||
return db.conn.Close()
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
schema := `
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
language TEXT NOT NULL,
|
||||
proto_repo TEXT NOT NULL,
|
||||
proto_branch TEXT NOT NULL,
|
||||
proto_commit TEXT NOT NULL DEFAULT '',
|
||||
target_repo TEXT NOT NULL,
|
||||
toolchain_image TEXT NOT NULL,
|
||||
image_digest TEXT NOT NULL DEFAULT '',
|
||||
result_commit TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
log TEXT NOT NULL DEFAULT '',
|
||||
retries INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_running_per_repo
|
||||
ON tasks(target_repo) WHERE status = 'running';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS toolchain_versions (
|
||||
language TEXT PRIMARY KEY,
|
||||
image_tag TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
`
|
||||
|
||||
_, err := db.conn.Exec(schema)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) Conn() *sql.DB {
|
||||
return db.conn
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"proto-gen/internal/models"
|
||||
)
|
||||
|
||||
func (db *DB) CreateTask(task *models.Task) error {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
_, err := db.conn.Exec(`
|
||||
INSERT INTO tasks (id, status, language, proto_repo, proto_branch, target_repo, toolchain_image, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
task.ID, task.Status, task.Language, task.ProtoRepo, task.ProtoBranch,
|
||||
task.TargetRepo, task.ToolchainImage, now, now,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetTask(id string) (*models.Task, error) {
|
||||
var t models.Task
|
||||
var createdAt, updatedAt string
|
||||
err := db.conn.QueryRow(`
|
||||
SELECT id, status, language, proto_repo, proto_branch, proto_commit, target_repo,
|
||||
toolchain_image, image_digest, result_commit, error, log, retries, created_at, updated_at
|
||||
FROM tasks WHERE id = ?`, id,
|
||||
).Scan(&t.ID, &t.Status, &t.Language, &t.ProtoRepo, &t.ProtoBranch, &t.ProtoCommit,
|
||||
&t.TargetRepo, &t.ToolchainImage, &t.ImageDigest, &t.ResultCommit,
|
||||
&t.Error, &t.Log, &t.Retries, &createdAt, &updatedAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("task not found: %s", id)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
|
||||
t.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt)
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
func (db *DB) UpdateTask(task *models.Task) error {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
_, err := db.conn.Exec(`
|
||||
UPDATE tasks SET status=?, proto_commit=?, image_digest=?, result_commit=?,
|
||||
error=?, log=?, retries=?, updated_at=?
|
||||
WHERE id=?`,
|
||||
task.Status, task.ProtoCommit, task.ImageDigest, task.ResultCommit,
|
||||
task.Error, task.Log, task.Retries, now, task.ID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListTasks(status string, limit, offset int) ([]*models.Task, error) {
|
||||
query := `SELECT id, status, language, proto_repo, proto_branch, proto_commit, target_repo,
|
||||
toolchain_image, image_digest, result_commit, error, log, retries, created_at, updated_at
|
||||
FROM tasks`
|
||||
var args []interface{}
|
||||
if status != "" {
|
||||
query += ` WHERE status = ?`
|
||||
args = append(args, status)
|
||||
}
|
||||
query += ` ORDER BY created_at DESC LIMIT ? OFFSET ?`
|
||||
args = append(args, limit, offset)
|
||||
|
||||
rows, err := db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var tasks []*models.Task
|
||||
for rows.Next() {
|
||||
var t models.Task
|
||||
var createdAt, updatedAt string
|
||||
if err := rows.Scan(&t.ID, &t.Status, &t.Language, &t.ProtoRepo, &t.ProtoBranch,
|
||||
&t.ProtoCommit, &t.TargetRepo, &t.ToolchainImage, &t.ImageDigest,
|
||||
&t.ResultCommit, &t.Error, &t.Log, &t.Retries, &createdAt, &updatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
|
||||
t.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt)
|
||||
tasks = append(tasks, &t)
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetPendingTasks(limit int) ([]*models.Task, error) {
|
||||
return db.ListTasks(string(models.StatusPending), limit, 0)
|
||||
}
|
||||
|
||||
func (db *DB) HasRunningTask(targetRepo string) (bool, error) {
|
||||
var count int
|
||||
err := db.conn.QueryRow(`SELECT COUNT(*) FROM tasks WHERE target_repo = ? AND status = 'running'`, targetRepo).Scan(&count)
|
||||
return count > 0, err
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"proto-gen/internal/models"
|
||||
)
|
||||
|
||||
func newTestDB(t *testing.T) *DB {
|
||||
t.Helper()
|
||||
db, err := New(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatalf("creating test db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestNew_CreatesTables(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
var count int
|
||||
err := db.conn.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('tasks', 'toolchain_versions')`).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("querying tables: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("expected 2 tables, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateAndGetTask(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
task := &models.Task{
|
||||
ID: "test-1",
|
||||
Status: models.StatusPending,
|
||||
Language: "go",
|
||||
ProtoRepo: "yoresee_doc",
|
||||
ProtoBranch: "main",
|
||||
TargetRepo: "yoresee_doc-gen-go",
|
||||
ToolchainImage: "proto-gen-go:v1.0",
|
||||
}
|
||||
if err := db.CreateTask(task); err != nil {
|
||||
t.Fatalf("CreateTask: %v", err)
|
||||
}
|
||||
|
||||
got, err := db.GetTask("test-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTask: %v", err)
|
||||
}
|
||||
if got.Language != "go" {
|
||||
t.Errorf("Language = %q, want %q", got.Language, "go")
|
||||
}
|
||||
if got.Status != models.StatusPending {
|
||||
t.Errorf("Status = %q, want %q", got.Status, models.StatusPending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasRunningTask(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
|
||||
has, err := db.HasRunningTask("yoresee_doc-gen-go")
|
||||
if err != nil {
|
||||
t.Fatalf("HasRunningTask: %v", err)
|
||||
}
|
||||
if has {
|
||||
t.Error("expected no running task")
|
||||
}
|
||||
|
||||
task := &models.Task{
|
||||
ID: "test-2",
|
||||
Status: models.StatusRunning,
|
||||
Language: "go",
|
||||
ProtoRepo: "yoresee_doc",
|
||||
ProtoBranch: "main",
|
||||
TargetRepo: "yoresee_doc-gen-go",
|
||||
ToolchainImage: "proto-gen-go:v1.0",
|
||||
}
|
||||
if err := db.CreateTask(task); err != nil {
|
||||
t.Fatalf("CreateTask: %v", err)
|
||||
}
|
||||
|
||||
has, err = db.HasRunningTask("yoresee_doc-gen-go")
|
||||
if err != nil {
|
||||
t.Fatalf("HasRunningTask: %v", err)
|
||||
}
|
||||
if !has {
|
||||
t.Error("expected running task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTasks(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
status := models.StatusPending
|
||||
if i == 1 {
|
||||
status = models.StatusRunning
|
||||
}
|
||||
db.CreateTask(&models.Task{
|
||||
ID: "t-" + string(rune('0'+i)),
|
||||
Status: status,
|
||||
Language: "go",
|
||||
ProtoRepo: "yoresee_doc",
|
||||
ProtoBranch: "main",
|
||||
TargetRepo: "repo-" + string(rune('0'+i)),
|
||||
ToolchainImage: "img:v1",
|
||||
})
|
||||
}
|
||||
|
||||
tasks, err := db.ListTasks("", 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasks: %v", err)
|
||||
}
|
||||
if len(tasks) != 3 {
|
||||
t.Errorf("len(tasks) = %d, want 3", len(tasks))
|
||||
}
|
||||
|
||||
tasks, err = db.ListTasks(string(models.StatusRunning), 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasks filtered: %v", err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Errorf("len(running) = %d, want 1", len(tasks))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package gitclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type giteaClient struct {
|
||||
baseURL string
|
||||
token string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewClient(baseURL, token string) GitClient {
|
||||
return &giteaClient{
|
||||
baseURL: baseURL,
|
||||
token: token,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *giteaClient) doRequest(method, path string, body interface{}) ([]byte, error) {
|
||||
u := c.baseURL + "/api/v1" + path
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshaling body: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(data)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, u, bodyReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "token "+c.token)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executing request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("gitea API error (%d): %s", resp.StatusCode, string(data))
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *giteaClient) ListRepositories(owner string) ([]Repository, error) {
|
||||
data, err := c.doRequest("GET", "/repos/search?limit=50&owner="+url.QueryEscape(owner), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Data []Repository `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
return result.Data, nil
|
||||
}
|
||||
|
||||
func (c *giteaClient) CreateRepository(owner, name, description string, private bool) (*Repository, error) {
|
||||
body := map[string]interface{}{
|
||||
"name": name,
|
||||
"description": description,
|
||||
"private": private,
|
||||
}
|
||||
data, err := c.doRequest("POST", "/user/repos", body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var repo Repository
|
||||
if err := json.Unmarshal(data, &repo); err != nil {
|
||||
return nil, fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
return &repo, nil
|
||||
}
|
||||
|
||||
func (c *giteaClient) GetRepository(owner, name string) (*Repository, error) {
|
||||
data, err := c.doRequest("GET", "/repos/"+url.PathEscape(owner)+"/"+url.PathEscape(name), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var repo Repository
|
||||
if err := json.Unmarshal(data, &repo); err != nil {
|
||||
return nil, fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
return &repo, nil
|
||||
}
|
||||
|
||||
func (c *giteaClient) ListBranches(owner, repo string) ([]Branch, error) {
|
||||
data, err := c.doRequest("GET", "/repos/"+url.PathEscape(owner)+"/"+url.PathEscape(repo)+"/branches", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var branches []Branch
|
||||
if err := json.Unmarshal(data, &branches); err != nil {
|
||||
return nil, fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
return branches, nil
|
||||
}
|
||||
|
||||
func (c *giteaClient) GetBranch(owner, repo, branch string) (*Branch, error) {
|
||||
data, err := c.doRequest("GET", "/repos/"+url.PathEscape(owner)+"/"+url.PathEscape(repo)+"/branches/"+url.PathEscape(branch), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b Branch
|
||||
if err := json.Unmarshal(data, &b); err != nil {
|
||||
return nil, fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package gitclient
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetBranch(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/repos/test-owner/test-repo/branches/main" {
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("Authorization") != "token test-token" {
|
||||
t.Errorf("unexpected auth header: %s", r.Header.Get("Authorization"))
|
||||
}
|
||||
json.NewEncoder(w).Encode(Branch{Name: "main", Commit: "abc123"})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "test-token")
|
||||
b, err := client.GetBranch("test-owner", "test-repo", "main")
|
||||
if err != nil {
|
||||
t.Fatalf("GetBranch: %v", err)
|
||||
}
|
||||
if b.Name != "main" {
|
||||
t.Errorf("Name = %q, want %q", b.Name, "main")
|
||||
}
|
||||
if b.Commit != "abc123" {
|
||||
t.Errorf("Commit = %q, want %q", b.Commit, "abc123")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListBranches(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode([]Branch{
|
||||
{Name: "main", Commit: "abc"},
|
||||
{Name: "dev", Commit: "def"},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "test-token")
|
||||
branches, err := client.ListBranches("owner", "repo")
|
||||
if err != nil {
|
||||
t.Fatalf("ListBranches: %v", err)
|
||||
}
|
||||
if len(branches) != 2 {
|
||||
t.Errorf("len(branches) = %d, want 2", len(branches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRepository(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
t.Errorf("unexpected method: %s", r.Method)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
if body["name"] != "new-repo" {
|
||||
t.Errorf("name = %v, want new-repo", body["name"])
|
||||
}
|
||||
json.NewEncoder(w).Encode(Repository{Name: "new-repo", FullName: "owner/new-repo"})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "test-token")
|
||||
repo, err := client.CreateRepository("owner", "new-repo", "test", false)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateRepository: %v", err)
|
||||
}
|
||||
if repo.Name != "new-repo" {
|
||||
t.Errorf("Name = %q, want %q", repo.Name, "new-repo")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package gitclient
|
||||
|
||||
type Repository struct {
|
||||
Name string `json:"name"`
|
||||
FullName string `json:"full_name"`
|
||||
Private bool `json:"private"`
|
||||
Empty bool `json:"empty"`
|
||||
}
|
||||
|
||||
type Branch struct {
|
||||
Name string `json:"name"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
type GitClient interface {
|
||||
ListRepositories(owner string) ([]Repository, error)
|
||||
CreateRepository(owner, name, description string, private bool) (*Repository, error)
|
||||
GetRepository(owner, name string) (*Repository, error)
|
||||
ListBranches(owner, repo string) ([]Branch, error)
|
||||
GetBranch(owner, repo, branch string) (*Branch, error)
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
type k8sClient struct {
|
||||
clientset kubernetes.Interface
|
||||
}
|
||||
|
||||
func NewClient(kubeconfig string) (K8sClient, error) {
|
||||
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("building config: %w", err)
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating clientset: %w", err)
|
||||
}
|
||||
|
||||
return &k8sClient{clientset: clientset}, nil
|
||||
}
|
||||
|
||||
func NewClientFromInterface(clientset kubernetes.Interface) K8sClient {
|
||||
return &k8sClient{clientset: clientset}
|
||||
}
|
||||
|
||||
func (c *k8sClient) CreateJob(spec *JobSpec) (*Job, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
envVars := make([]corev1.EnvVar, 0, len(spec.EnvVars))
|
||||
for k, v := range spec.EnvVars {
|
||||
envVars = append(envVars, corev1.EnvVar{Name: k, Value: v})
|
||||
}
|
||||
|
||||
backoffLimit := int32(0)
|
||||
job := &batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: spec.Name,
|
||||
Namespace: spec.Namespace,
|
||||
},
|
||||
Spec: batchv1.JobSpec{
|
||||
BackoffLimit: &backoffLimit,
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "toolchain",
|
||||
Image: spec.Image,
|
||||
Env: envVars,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if spec.TimeoutSecond > 0 {
|
||||
timeout := int64(spec.TimeoutSecond)
|
||||
job.Spec.ActiveDeadlineSeconds = &timeout
|
||||
}
|
||||
|
||||
result, err := c.clientset.BatchV1().Jobs(spec.Namespace).Create(ctx, job, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating job: %w", err)
|
||||
}
|
||||
|
||||
return &Job{
|
||||
Name: result.Name,
|
||||
Namespace: result.Namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *k8sClient) GetJob(name, namespace string) (*Job, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
result, err := c.clientset.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting job: %w", err)
|
||||
}
|
||||
|
||||
status := JobPending
|
||||
if result.Status.Succeeded > 0 {
|
||||
status = JobSucceeded
|
||||
} else if result.Status.Failed > 0 {
|
||||
status = JobFailed
|
||||
} else if result.Status.Active > 0 {
|
||||
status = JobRunning
|
||||
}
|
||||
|
||||
job := &Job{
|
||||
Name: result.Name,
|
||||
Namespace: result.Namespace,
|
||||
Status: status,
|
||||
}
|
||||
|
||||
if status == JobFailed {
|
||||
logs, _ := c.getPodLogs(name, namespace)
|
||||
job.Logs = logs
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (c *k8sClient) DeleteJob(name, namespace string) error {
|
||||
ctx := context.Background()
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
return c.clientset.BatchV1().Jobs(namespace).Delete(ctx, name, metav1.DeleteOptions{
|
||||
PropagationPolicy: &propagation,
|
||||
})
|
||||
}
|
||||
|
||||
func (c *k8sClient) getPodLogs(jobName, namespace string) (string, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
pods, err := c.clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("job-name=%s", jobName),
|
||||
})
|
||||
if err != nil || len(pods.Items) == 0 {
|
||||
return "", err
|
||||
}
|
||||
|
||||
pod := pods.Items[0]
|
||||
req := c.clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{})
|
||||
stream, err := req.Stream(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer stream.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
if _, err := io.Copy(&buf, stream); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestCreateJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.CreateJob(&JobSpec{
|
||||
Name: "test-job",
|
||||
Namespace: "default",
|
||||
Image: "test-image:v1",
|
||||
EnvVars: map[string]string{"KEY": "value"},
|
||||
TimeoutSecond: 300,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateJob: %v", err)
|
||||
}
|
||||
if job.Name != "test-job" {
|
||||
t.Errorf("Name = %q, want %q", job.Name, "test-job")
|
||||
}
|
||||
if job.Status != JobPending {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobPending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "done-job", Namespace: "default"},
|
||||
Status: batchv1.JobStatus{Succeeded: 1},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.GetJob("done-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("GetJob: %v", err)
|
||||
}
|
||||
if job.Status != JobSucceeded {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobSucceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetJob_Running(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "run-job", Namespace: "default"},
|
||||
Status: batchv1.JobStatus{Active: 1},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.GetJob("run-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("GetJob: %v", err)
|
||||
}
|
||||
if job.Status != JobRunning {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobRunning)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "del-job", Namespace: "default"},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
err := client.DeleteJob("del-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteJob: %v", err)
|
||||
}
|
||||
|
||||
_, err = clientset.BatchV1().Jobs("default").Get(nil, "del-job", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("expected error getting deleted job")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package k8s
|
||||
|
||||
type JobStatus string
|
||||
|
||||
const (
|
||||
JobPending JobStatus = "Pending"
|
||||
JobRunning JobStatus = "Running"
|
||||
JobSucceeded JobStatus = "Succeeded"
|
||||
JobFailed JobStatus = "Failed"
|
||||
)
|
||||
|
||||
type JobSpec struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Image string
|
||||
EnvVars map[string]string
|
||||
TimeoutSecond int
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Status JobStatus
|
||||
Logs string
|
||||
}
|
||||
|
||||
type K8sClient interface {
|
||||
CreateJob(spec *JobSpec) (*Job, error)
|
||||
GetJob(name, namespace string) (*Job, error)
|
||||
DeleteJob(name, namespace string) error
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package k8s
|
||||
|
||||
type noopClient struct{}
|
||||
|
||||
func NewNoopClient() K8sClient {
|
||||
return &noopClient{}
|
||||
}
|
||||
|
||||
func (c *noopClient) CreateJob(spec *JobSpec) (*Job, error) {
|
||||
return &Job{
|
||||
Name: spec.Name,
|
||||
Namespace: spec.Namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *noopClient) GetJob(name, namespace string) (*Job, error) {
|
||||
return &Job{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *noopClient) DeleteJob(name, namespace string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type TaskStatus string
|
||||
|
||||
const (
|
||||
StatusPending TaskStatus = "pending"
|
||||
StatusRunning TaskStatus = "running"
|
||||
StatusSuccess TaskStatus = "success"
|
||||
StatusFailed TaskStatus = "failed"
|
||||
)
|
||||
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Status TaskStatus `json:"status"`
|
||||
Language string `json:"language"`
|
||||
ProtoRepo string `json:"proto_repo"`
|
||||
ProtoBranch string `json:"proto_branch"`
|
||||
ProtoCommit string `json:"proto_commit,omitempty"`
|
||||
TargetRepo string `json:"target_repo"`
|
||||
ToolchainImage string `json:"toolchain_image"`
|
||||
ImageDigest string `json:"image_digest,omitempty"`
|
||||
ResultCommit string `json:"result_commit,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Log string `json:"log,omitempty"`
|
||||
Retries int `json:"retries"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ToolchainVersion struct {
|
||||
Language string `json:"language"`
|
||||
ImageTag string `json:"image_tag"`
|
||||
Enabled bool `json:"enabled"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
const API_BASE = '/api/v1';
|
||||
|
||||
async function api(method, path, body) {
|
||||
const options = {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
};
|
||||
if (body) options.body = JSON.stringify(body);
|
||||
const res = await fetch(API_BASE + path, options);
|
||||
if (!res.ok) {
|
||||
const err = await res.json();
|
||||
throw new Error(err.error || 'Request failed');
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function showMessage(text, type = 'success') {
|
||||
const msg = document.getElementById('message');
|
||||
if (!msg) return;
|
||||
msg.textContent = text;
|
||||
msg.className = `message ${type}`;
|
||||
setTimeout(() => msg.className = 'hidden', 5000);
|
||||
}
|
||||
|
||||
function formatDate(dateStr) {
|
||||
if (!dateStr) return '-';
|
||||
return new Date(dateStr).toLocaleString();
|
||||
}
|
||||
|
||||
function statusBadge(status) {
|
||||
const colors = {
|
||||
pending: '#f39c12',
|
||||
running: '#3498db',
|
||||
success: '#27ae60',
|
||||
failed: '#e74c3c'
|
||||
};
|
||||
return `<span class="badge" style="background:${colors[status] || '#999'}">${status}</span>`;
|
||||
}
|
||||
|
||||
async function initGeneratePage() {
|
||||
try {
|
||||
const plugins = await api('GET', '/plugins');
|
||||
const select = document.getElementById('language');
|
||||
plugins.forEach(p => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = p.language;
|
||||
opt.textContent = p.language;
|
||||
select.appendChild(opt);
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to load plugins:', e);
|
||||
}
|
||||
|
||||
document.getElementById('generateForm').addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
const btn = e.target.querySelector('button[type="submit"]');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Generating...';
|
||||
|
||||
try {
|
||||
const task = await api('POST', '/tasks', {
|
||||
language: document.getElementById('language').value,
|
||||
proto_repo: document.getElementById('protoRepo').value,
|
||||
proto_branch: document.getElementById('protoBranch').value,
|
||||
target_repo: document.getElementById('targetRepo').value || undefined
|
||||
});
|
||||
showMessage(`Task created: ${task.id}`, 'success');
|
||||
e.target.reset();
|
||||
document.getElementById('protoBranch').value = 'main';
|
||||
} catch (err) {
|
||||
showMessage(err.message, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Generate';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function initTasksPage() {
|
||||
await refreshTasks();
|
||||
document.getElementById('statusFilter').addEventListener('change', refreshTasks);
|
||||
setInterval(refreshTasks, 10000);
|
||||
}
|
||||
|
||||
async function refreshTasks() {
|
||||
const status = document.getElementById('statusFilter').value;
|
||||
const params = status ? `?status=${status}` : '';
|
||||
try {
|
||||
const tasks = await api('GET', `/tasks${params}`);
|
||||
const tbody = document.getElementById('tasksBody');
|
||||
const noTasks = document.getElementById('noTasks');
|
||||
|
||||
if (!tasks || tasks.length === 0) {
|
||||
tbody.innerHTML = '';
|
||||
noTasks.classList.remove('hidden');
|
||||
return;
|
||||
}
|
||||
|
||||
noTasks.classList.add('hidden');
|
||||
tbody.innerHTML = tasks.map(t => `
|
||||
<tr>
|
||||
<td><code>${t.id.substring(0, 8)}</code></td>
|
||||
<td>${t.language}</td>
|
||||
<td>${t.proto_repo}</td>
|
||||
<td>${t.target_repo}</td>
|
||||
<td>${statusBadge(t.status)}</td>
|
||||
<td>${formatDate(t.created_at)}</td>
|
||||
<td>
|
||||
${t.status === 'failed' ? `<button onclick="retryTask('${t.id}')" class="btn-small">Retry</button>` : ''}
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
} catch (err) {
|
||||
console.error('Failed to load tasks:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async function retryTask(id) {
|
||||
try {
|
||||
await api('POST', `/tasks/${id}/retry`);
|
||||
showMessage('Task queued for retry');
|
||||
refreshTasks();
|
||||
} catch (err) {
|
||||
showMessage(err.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async function initSettingsPage() {
|
||||
try {
|
||||
const plugins = await api('GET', '/plugins');
|
||||
const container = document.getElementById('pluginsList');
|
||||
container.innerHTML = plugins.map(p => `
|
||||
<div class="plugin-card">
|
||||
<h3>${p.language}</h3>
|
||||
<p>Image: <code>${p.image}</code></p>
|
||||
</div>
|
||||
`).join('');
|
||||
} catch (err) {
|
||||
console.error('Failed to load plugins:', err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Proto-Gen Platform</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<nav>
|
||||
<a href="/" class="logo">Proto-Gen</a>
|
||||
<div class="nav-links">
|
||||
<a href="/" class="active">Generate</a>
|
||||
<a href="/tasks.html">Tasks</a>
|
||||
<a href="/settings.html">Settings</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
<h1>Generate Code</h1>
|
||||
<form id="generateForm">
|
||||
<div class="form-group">
|
||||
<label for="protoRepo">Proto Repository</label>
|
||||
<input type="text" id="protoRepo" placeholder="e.g., owner/yoresee_doc" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="protoBranch">Branch</label>
|
||||
<input type="text" id="protoBranch" value="main" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="language">Target Language</label>
|
||||
<select id="language" required>
|
||||
<option value="">Select language...</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="targetRepo">Target Repository</label>
|
||||
<input type="text" id="targetRepo" placeholder="Auto-generated if empty">
|
||||
<small>Leave empty to auto-generate: {proto_repo}-gen-{lang}</small>
|
||||
</div>
|
||||
|
||||
<button type="submit" class="btn-primary">Generate</button>
|
||||
</form>
|
||||
|
||||
<div id="message" class="hidden"></div>
|
||||
</main>
|
||||
|
||||
<script src="/app.js"></script>
|
||||
<script>
|
||||
initGeneratePage();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,30 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Settings - Proto-Gen Platform</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<nav>
|
||||
<a href="/" class="logo">Proto-Gen</a>
|
||||
<div class="nav-links">
|
||||
<a href="/">Generate</a>
|
||||
<a href="/tasks.html">Tasks</a>
|
||||
<a href="/settings.html" class="active">Settings</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
<h1>Settings</h1>
|
||||
<h2>Toolchain Versions</h2>
|
||||
<div id="pluginsList"></div>
|
||||
</main>
|
||||
|
||||
<script src="/app.js"></script>
|
||||
<script>
|
||||
initSettingsPage();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,191 @@
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: #f5f5f5;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
nav {
|
||||
background: #2c3e50;
|
||||
padding: 1rem 2rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.logo {
|
||||
color: #ecf0f1;
|
||||
font-size: 1.5rem;
|
||||
font-weight: bold;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.nav-links a {
|
||||
color: #bdc3c7;
|
||||
text-decoration: none;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.nav-links a:hover,
|
||||
.nav-links a.active {
|
||||
color: #ecf0f1;
|
||||
background: rgba(255,255,255,0.1);
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 900px;
|
||||
margin: 2rem auto;
|
||||
padding: 0 1rem;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin-bottom: 1.5rem;
|
||||
color: #2c3e50;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 1.5rem 0 1rem;
|
||||
color: #34495e;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
margin-bottom: 0.5rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
input, select {
|
||||
width: 100%;
|
||||
padding: 0.75rem;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
input:focus, select:focus {
|
||||
outline: none;
|
||||
border-color: #3498db;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #7f8c8d;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 0.75rem 1.5rem;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: #3498db;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-primary:hover {
|
||||
background: #2980b9;
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: #95a5a6;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-small {
|
||||
padding: 0.25rem 0.75rem;
|
||||
font-size: 0.85rem;
|
||||
background: #e74c3c;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 12px;
|
||||
color: white;
|
||||
font-size: 0.85rem;
|
||||
text-transform: capitalize;
|
||||
}
|
||||
|
||||
.filters {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.filters select {
|
||||
width: auto;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
|
||||
th, td {
|
||||
padding: 0.75rem 1rem;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
|
||||
th {
|
||||
background: #f8f9fa;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
code {
|
||||
background: #f0f0f0;
|
||||
padding: 0.15rem 0.4rem;
|
||||
border-radius: 3px;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.message {
|
||||
padding: 1rem;
|
||||
margin-top: 1rem;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.message.success {
|
||||
background: #d4edda;
|
||||
color: #155724;
|
||||
}
|
||||
|
||||
.message.error {
|
||||
background: #f8d7da;
|
||||
color: #721c24;
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.plugin-card {
|
||||
background: white;
|
||||
padding: 1rem;
|
||||
margin-bottom: 1rem;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
}
|
||||
|
||||
.plugin-card h3 {
|
||||
margin-bottom: 0.5rem;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Tasks - Proto-Gen Platform</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<nav>
|
||||
<a href="/" class="logo">Proto-Gen</a>
|
||||
<div class="nav-links">
|
||||
<a href="/">Generate</a>
|
||||
<a href="/tasks.html" class="active">Tasks</a>
|
||||
<a href="/settings.html">Settings</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
<h1>Tasks</h1>
|
||||
<div class="filters">
|
||||
<select id="statusFilter">
|
||||
<option value="">All Status</option>
|
||||
<option value="pending">Pending</option>
|
||||
<option value="running">Running</option>
|
||||
<option value="success">Success</option>
|
||||
<option value="failed">Failed</option>
|
||||
</select>
|
||||
<button onclick="refreshTasks()" class="btn-secondary">Refresh</button>
|
||||
</div>
|
||||
|
||||
<table id="tasksTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Language</th>
|
||||
<th>Proto Repo</th>
|
||||
<th>Target Repo</th>
|
||||
<th>Status</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="tasksBody">
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div id="noTasks" class="hidden">No tasks found</div>
|
||||
</main>
|
||||
|
||||
<script src="/app.js"></script>
|
||||
<script>
|
||||
initTasksPage();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,145 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"proto-gen/internal/config"
|
||||
"proto-gen/internal/database"
|
||||
"proto-gen/internal/k8s"
|
||||
"proto-gen/internal/models"
|
||||
)
|
||||
|
||||
type Scheduler struct {
|
||||
db *database.DB
|
||||
k8sClient k8s.K8sClient
|
||||
cfg *config.Config
|
||||
pollInterval time.Duration
|
||||
running map[string]bool
|
||||
}
|
||||
|
||||
func NewScheduler(db *database.DB, k8sClient k8s.K8sClient, cfg *config.Config) *Scheduler {
|
||||
return &Scheduler{
|
||||
db: db,
|
||||
k8sClient: k8sClient,
|
||||
cfg: cfg,
|
||||
pollInterval: 5 * time.Second,
|
||||
running: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) Start(ctx context.Context) {
|
||||
log.Println("scheduler started")
|
||||
ticker := time.NewTicker(s.pollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("scheduler stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.processPendingTasks()
|
||||
s.trackRunningJobs()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) processPendingTasks() {
|
||||
pendingTasks, err := s.db.GetPendingTasks(10)
|
||||
if err != nil {
|
||||
log.Printf("error getting pending tasks: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, task := range pendingTasks {
|
||||
if len(s.running) >= s.cfg.Cluster.MaxConcurrent {
|
||||
log.Printf("max concurrent reached, skipping task %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
hasRunning, err := s.db.HasRunningTask(task.TargetRepo)
|
||||
if err != nil {
|
||||
log.Printf("error checking running task: %v", err)
|
||||
continue
|
||||
}
|
||||
if hasRunning {
|
||||
log.Printf("task %s: target repo %s already has running task", task.ID, task.TargetRepo)
|
||||
continue
|
||||
}
|
||||
|
||||
s.startTask(task)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) startTask(task *models.Task) {
|
||||
envVars := map[string]string{
|
||||
"GIT_BASE_URL": s.cfg.Git.BaseURL,
|
||||
"GIT_TOKEN": s.cfg.Git.Token,
|
||||
"PROTO_REPO": task.ProtoRepo,
|
||||
"PROTO_BRANCH": task.ProtoBranch,
|
||||
"TARGET_REPO": task.TargetRepo,
|
||||
"OUT_DIR": "/out",
|
||||
}
|
||||
|
||||
jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
|
||||
job, err := s.k8sClient.CreateJob(&k8s.JobSpec{
|
||||
Name: jobName,
|
||||
Namespace: s.cfg.Cluster.Namespace,
|
||||
Image: task.ToolchainImage,
|
||||
EnvVars: envVars,
|
||||
TimeoutSecond: s.cfg.Cluster.JobTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("task %s: failed to create job: %v", task.ID, err)
|
||||
task.Status = models.StatusFailed
|
||||
task.Error = fmt.Sprintf("failed to create K8s job: %v", err)
|
||||
s.db.UpdateTask(task)
|
||||
return
|
||||
}
|
||||
|
||||
task.Status = models.StatusRunning
|
||||
if err := s.db.UpdateTask(task); err != nil {
|
||||
log.Printf("task %s: failed to update status: %v", task.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
s.running[task.ID] = true
|
||||
log.Printf("task %s: job %s created", task.ID, job.Name)
|
||||
}
|
||||
|
||||
func (s *Scheduler) trackRunningJobs() {
|
||||
for taskID := range s.running {
|
||||
task, err := s.db.GetTask(taskID)
|
||||
if err != nil {
|
||||
delete(s.running, taskID)
|
||||
continue
|
||||
}
|
||||
|
||||
jobName := fmt.Sprintf("proto-gen-%s", task.ID[:8])
|
||||
job, err := s.k8sClient.GetJob(jobName, s.cfg.Cluster.Namespace)
|
||||
if err != nil {
|
||||
log.Printf("task %s: error getting job: %v", taskID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
switch job.Status {
|
||||
case k8s.JobSucceeded:
|
||||
task.Status = models.StatusSuccess
|
||||
task.Log = job.Logs
|
||||
s.db.UpdateTask(task)
|
||||
delete(s.running, taskID)
|
||||
log.Printf("task %s: completed successfully", taskID)
|
||||
|
||||
case k8s.JobFailed:
|
||||
task.Status = models.StatusFailed
|
||||
task.Error = "K8s job failed"
|
||||
task.Log = job.Logs
|
||||
s.db.UpdateTask(task)
|
||||
delete(s.running, taskID)
|
||||
log.Printf("task %s: failed", taskID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proto-gen/internal/config"
|
||||
"proto-gen/internal/database"
|
||||
"proto-gen/internal/k8s"
|
||||
)
|
||||
|
||||
func newTestScheduler(t *testing.T) (*Scheduler, *database.DB) {
|
||||
t.Helper()
|
||||
db, err := database.New(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatalf("creating db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
cfg := &config.Config{
|
||||
Cluster: config.ClusterConfig{
|
||||
Namespace: "test",
|
||||
JobTimeout: 300,
|
||||
MaxConcurrent: 2,
|
||||
},
|
||||
ToolchainRegistry: "registry.example.com",
|
||||
}
|
||||
|
||||
k8sClient := k8s.NewClientFromInterface(nil)
|
||||
scheduler := NewScheduler(db, k8sClient, cfg)
|
||||
scheduler.pollInterval = 100 * time.Millisecond
|
||||
|
||||
return scheduler, db
|
||||
}
|
||||
|
||||
func TestNewScheduler(t *testing.T) {
|
||||
scheduler, db := newTestScheduler(t)
|
||||
defer db.Close()
|
||||
|
||||
if scheduler.db == nil {
|
||||
t.Error("db is nil")
|
||||
}
|
||||
if scheduler.k8sClient == nil {
|
||||
t.Error("k8sClient is nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessPendingTasks_Empty(t *testing.T) {
|
||||
scheduler, db := newTestScheduler(t)
|
||||
defer db.Close()
|
||||
|
||||
scheduler.processPendingTasks()
|
||||
}
|
||||
Reference in New Issue
Block a user