feat: implement proto-gen platform
- Config loader with env var expansion and validation - SQLite database with tasks and toolchain_versions tables - Gitea REST API client for repository and branch operations - K8s client for Job lifecycle management (with noop fallback for local dev) - REST API: tasks CRUD, retry, plugins, version management - Worker/scheduler with polling, concurrency control, and job tracking - Static frontend: generate, tasks, settings pages - Toolchain images: Go and Node.js Dockerfiles with entrypoint scripts - Helm chart with deployment, service, configmap, ingress, PVC - Multi-stage Dockerfile for service - Unit tests for config, database, gitclient, k8s, api, worker - E2E tests for idempotency, version switch, failure recovery
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
type k8sClient struct {
|
||||
clientset kubernetes.Interface
|
||||
}
|
||||
|
||||
func NewClient(kubeconfig string) (K8sClient, error) {
|
||||
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("building config: %w", err)
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating clientset: %w", err)
|
||||
}
|
||||
|
||||
return &k8sClient{clientset: clientset}, nil
|
||||
}
|
||||
|
||||
func NewClientFromInterface(clientset kubernetes.Interface) K8sClient {
|
||||
return &k8sClient{clientset: clientset}
|
||||
}
|
||||
|
||||
func (c *k8sClient) CreateJob(spec *JobSpec) (*Job, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
envVars := make([]corev1.EnvVar, 0, len(spec.EnvVars))
|
||||
for k, v := range spec.EnvVars {
|
||||
envVars = append(envVars, corev1.EnvVar{Name: k, Value: v})
|
||||
}
|
||||
|
||||
backoffLimit := int32(0)
|
||||
job := &batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: spec.Name,
|
||||
Namespace: spec.Namespace,
|
||||
},
|
||||
Spec: batchv1.JobSpec{
|
||||
BackoffLimit: &backoffLimit,
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "toolchain",
|
||||
Image: spec.Image,
|
||||
Env: envVars,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if spec.TimeoutSecond > 0 {
|
||||
timeout := int64(spec.TimeoutSecond)
|
||||
job.Spec.ActiveDeadlineSeconds = &timeout
|
||||
}
|
||||
|
||||
result, err := c.clientset.BatchV1().Jobs(spec.Namespace).Create(ctx, job, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating job: %w", err)
|
||||
}
|
||||
|
||||
return &Job{
|
||||
Name: result.Name,
|
||||
Namespace: result.Namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *k8sClient) GetJob(name, namespace string) (*Job, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
result, err := c.clientset.BatchV1().Jobs(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting job: %w", err)
|
||||
}
|
||||
|
||||
status := JobPending
|
||||
if result.Status.Succeeded > 0 {
|
||||
status = JobSucceeded
|
||||
} else if result.Status.Failed > 0 {
|
||||
status = JobFailed
|
||||
} else if result.Status.Active > 0 {
|
||||
status = JobRunning
|
||||
}
|
||||
|
||||
job := &Job{
|
||||
Name: result.Name,
|
||||
Namespace: result.Namespace,
|
||||
Status: status,
|
||||
}
|
||||
|
||||
if status == JobFailed {
|
||||
logs, _ := c.getPodLogs(name, namespace)
|
||||
job.Logs = logs
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (c *k8sClient) DeleteJob(name, namespace string) error {
|
||||
ctx := context.Background()
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
return c.clientset.BatchV1().Jobs(namespace).Delete(ctx, name, metav1.DeleteOptions{
|
||||
PropagationPolicy: &propagation,
|
||||
})
|
||||
}
|
||||
|
||||
func (c *k8sClient) getPodLogs(jobName, namespace string) (string, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
pods, err := c.clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("job-name=%s", jobName),
|
||||
})
|
||||
if err != nil || len(pods.Items) == 0 {
|
||||
return "", err
|
||||
}
|
||||
|
||||
pod := pods.Items[0]
|
||||
req := c.clientset.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{})
|
||||
stream, err := req.Stream(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer stream.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
if _, err := io.Copy(&buf, stream); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestCreateJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.CreateJob(&JobSpec{
|
||||
Name: "test-job",
|
||||
Namespace: "default",
|
||||
Image: "test-image:v1",
|
||||
EnvVars: map[string]string{"KEY": "value"},
|
||||
TimeoutSecond: 300,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateJob: %v", err)
|
||||
}
|
||||
if job.Name != "test-job" {
|
||||
t.Errorf("Name = %q, want %q", job.Name, "test-job")
|
||||
}
|
||||
if job.Status != JobPending {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobPending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "done-job", Namespace: "default"},
|
||||
Status: batchv1.JobStatus{Succeeded: 1},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.GetJob("done-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("GetJob: %v", err)
|
||||
}
|
||||
if job.Status != JobSucceeded {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobSucceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetJob_Running(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "run-job", Namespace: "default"},
|
||||
Status: batchv1.JobStatus{Active: 1},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
job, err := client.GetJob("run-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("GetJob: %v", err)
|
||||
}
|
||||
if job.Status != JobRunning {
|
||||
t.Errorf("Status = %q, want %q", job.Status, JobRunning)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteJob(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset(&batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "del-job", Namespace: "default"},
|
||||
})
|
||||
client := NewClientFromInterface(clientset)
|
||||
|
||||
err := client.DeleteJob("del-job", "default")
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteJob: %v", err)
|
||||
}
|
||||
|
||||
_, err = clientset.BatchV1().Jobs("default").Get(nil, "del-job", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("expected error getting deleted job")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package k8s
|
||||
|
||||
type JobStatus string
|
||||
|
||||
const (
|
||||
JobPending JobStatus = "Pending"
|
||||
JobRunning JobStatus = "Running"
|
||||
JobSucceeded JobStatus = "Succeeded"
|
||||
JobFailed JobStatus = "Failed"
|
||||
)
|
||||
|
||||
type JobSpec struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Image string
|
||||
EnvVars map[string]string
|
||||
TimeoutSecond int
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Status JobStatus
|
||||
Logs string
|
||||
}
|
||||
|
||||
type K8sClient interface {
|
||||
CreateJob(spec *JobSpec) (*Job, error)
|
||||
GetJob(name, namespace string) (*Job, error)
|
||||
DeleteJob(name, namespace string) error
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package k8s
|
||||
|
||||
type noopClient struct{}
|
||||
|
||||
func NewNoopClient() K8sClient {
|
||||
return &noopClient{}
|
||||
}
|
||||
|
||||
func (c *noopClient) CreateJob(spec *JobSpec) (*Job, error) {
|
||||
return &Job{
|
||||
Name: spec.Name,
|
||||
Namespace: spec.Namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *noopClient) GetJob(name, namespace string) (*Job, error) {
|
||||
return &Job{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Status: JobPending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *noopClient) DeleteJob(name, namespace string) error {
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user