From 72a526c65aee6cbb15db71dd71ea1ca19df6a34f Mon Sep 17 00:00:00 2001 From: panshuxiao Date: Wed, 28 May 2025 01:30:51 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=BA=86webterminal=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devcontainerapp_controller.go | 86 +++++++ services/devcontainer/devcontainer.go | 102 +++++++-- services/devcontainer/k8s_agent.go | 211 +++++++++++++++++- 3 files changed, 377 insertions(+), 22 deletions(-) diff --git a/modules/k8s/controller/devcontainer/devcontainerapp_controller.go b/modules/k8s/controller/devcontainer/devcontainerapp_controller.go index 5ff8110cec..3a634523f9 100644 --- a/modules/k8s/controller/devcontainer/devcontainerapp_controller.go +++ b/modules/k8s/controller/devcontainer/devcontainerapp_controller.go @@ -19,6 +19,7 @@ package devcontainer import ( "context" "strconv" + "strings" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -46,6 +47,7 @@ type DevcontainerAppReconciler struct { // +kubebuilder:rbac:groups=devcontainer.devstar.cn,resources=devcontainerapps/finalizers,verbs=update // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=create;delete;get;list;watch // +kubebuilder:rbac:groups="",resources=services,verbs=create;delete;get;list;watch +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -69,6 +71,44 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, client.IgnoreNotFound(err) } + // 添加 finalizer 处理逻辑 + finalizerName := "devcontainer.devstar.cn/finalizer" + + // 检查对象是否正在被删除 + if !app.ObjectMeta.DeletionTimestamp.IsZero() { + // 对象正在被删除 - 处理 finalizer + if k8s_sigs_controller_runtime_utils.ContainsFinalizer(app, finalizerName) { + // 执行清理操作 + logger.Info("Cleaning up resources before deletion", "name", app.Name) + + // 查找并删除关联的 PVC + if err := r.cleanupPersistentVolumeClaims(ctx, app); err != nil { + logger.Error(err, "Failed to clean up PVCs") + return ctrl.Result{}, err + } + + // 删除完成后移除 finalizer + k8s_sigs_controller_runtime_utils.RemoveFinalizer(app, finalizerName) + if err := r.Update(ctx, app); err != nil { + logger.Error(err, "Failed to remove finalizer") + return ctrl.Result{}, err + } + } + + // 已标记为删除且处理完成，允许继续删除流程 + return ctrl.Result{}, nil + } + + // 如果对象不包含 finalizer，就添加它 + if !k8s_sigs_controller_runtime_utils.ContainsFinalizer(app, finalizerName) { + logger.Info("Adding finalizer", "name", app.Name) + k8s_sigs_controller_runtime_utils.AddFinalizer(app, finalizerName) + if err := r.Update(ctx, app); err != nil { + logger.Error(err, "Failed to add finalizer") + return ctrl.Result{}, err + } + } + // 检查停止容器的注解 if desiredReplicas, exists := app.Annotations["devstar.io/desiredReplicas"]; exists && desiredReplicas == "0" { logger.Info("DevContainer stop requested via annotation", "name", app.Name) @@ -353,6 +393,52 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } +// cleanupPersistentVolumeClaims 查找并删除与 DevcontainerApp 关联的所有 PVC +func (r *DevcontainerAppReconciler) cleanupPersistentVolumeClaims(ctx context.Context, app *devcontainer_v1.DevcontainerApp) error { + logger := log.FromContext(ctx) + + // 查找关联的 PVC + pvcList := &core_v1.PersistentVolumeClaimList{} + + // 按标签筛选 + labelSelector := client.MatchingLabels{ + "app": app.Name, + } + if err := r.List(ctx, pvcList, client.InNamespace(app.Namespace), labelSelector); err != nil { + return err + } + + // 如果按标签没找到，尝试按名称模式查找 + if len(pvcList.Items) == 0 { + if err := r.List(ctx, pvcList, client.InNamespace(app.Namespace)); err != nil { + return err + } + + // 筛选出名称包含 DevcontainerApp 名称的 PVC + var filteredItems []core_v1.PersistentVolumeClaim + for _, pvc := range pvcList.Items { + // StatefulSet PVC 命名格式通常为: --<序号> + // 检查是否包含 app 名称作为名称的一部分 + if strings.Contains(pvc.Name, app.Name+"-") { + filteredItems = append(filteredItems, pvc) + logger.Info("Found PVC to delete", "name", pvc.Name) + } + } + pvcList.Items = filteredItems + } + + // 删除找到的 PVC + for i := range pvcList.Items { + logger.Info("Deleting PVC", "name", pvcList.Items[i].Name) + if err := r.Delete(ctx, &pvcList.Items[i]); err != nil && !errors.IsNotFound(err) { + logger.Error(err, "Failed to delete PVC", "name", pvcList.Items[i].Name) + return err + } + } + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/services/devcontainer/devcontainer.go b/services/devcontainer/devcontainer.go index 2fd6e5579b..8209cd909b 100644 --- a/services/devcontainer/devcontainer.go +++ b/services/devcontainer/devcontainer.go @@ -409,9 +409,25 @@ func GetWebTerminalURL(ctx context.Context, devcontainerName string) (string, er log.Error("GetWebTerminalURL: 加载配置文件失败: %v", err) return "", err } + + // 检查是否启用了基于路径的访问方式 + domain := cfg.Section("server").Key("DOMAIN").Value() - log.Info("GetWebTerminalURL: 生成 ttyd URL: http://%s:%d/", domain, ttydNodePort) - return fmt.Sprintf("http://%s:%d/", domain, ttydNodePort), nil + scheme := "https" + + // 从容器名称中提取用户名和仓库名 + parts := strings.Split(devcontainerName, "-") + if len(parts) >= 2 { + username := parts[0] + repoName := parts[1] + + // 构建访问路径 + path := fmt.Sprintf("/%s/%s/dev-container-webterminal", username, repoName) + terminalURL := fmt.Sprintf("%s://%s%s", scheme, domain, path) + + log.Info("GetWebTerminalURL: 使用 Ingress 路径方式生成 ttyd URL: %s", terminalURL) + return terminalURL, nil + } } // 如果没有找到ttyd端口，记录详细的调试信息 @@ -444,26 +460,12 @@ func GetWebTerminalURL(ctx context.Context, devcontainerName string) (string, er } func Get_IDE_TerminalURL(ctx *gitea_context.Context, devcontainer *RepoDevContainer) (string, error) { var access_token string - defalut_ctx := context.Background() - // 获取端口号 - cli, err := docker.CreateDockerClient(&defalut_ctx) - if err != nil { - return "", err - } - defer cli.Close() - containerID, err := docker.GetContainerID(cli, devcontainer.DevContainerName) - if err != nil { - return "", err - } - port, err := docker.GetMappedPort(cli, containerID, "22") - if err != nil { - return "", err - } - // 检查session + + // 检查 session 中是否已存在 token if ctx.Session.Get("access_token") != nil { access_token = ctx.Session.Get("access_token").(string) } else { - // 生成token + // 生成 token token := &auth_model.AccessToken{ UID: devcontainer.UserId, Name: "terminal_login_token", @@ -486,8 +488,67 @@ func Get_IDE_TerminalURL(ctx *gitea_context.Context, devcontainer *RepoDevContai } ctx.Session.Set("terminal_login_token", token.Token) access_token = token.Token - } + + // 根据不同的代理类型获取 SSH 端口 + var port string + + switch setting.Devcontainer.Agent { + case setting.KUBERNETES, "k8s": + // 创建 K8s 客户端 + apiRequestContext := ctx.Req.Context() + k8sClient, err := devcontainer_k8s_agent_module.GetKubernetesClient(&apiRequestContext) + if err != nil { + log.Error("Get_IDE_TerminalURL: 创建 K8s 客户端失败: %v", err) + return "", err + } + + // 获取 DevcontainerApp 资源 + opts := &devcontainer_k8s_agent_module.GetDevcontainerOptions{ + GetOptions: metav1.GetOptions{}, + Name: devcontainer.DevContainerName, + Namespace: setting.Devcontainer.Namespace, + Wait: false, + } + + log.Info("Get_IDE_TerminalURL: 从 K8s 获取 DevcontainerApp %s, namespace=%s", + devcontainer.DevContainerName, setting.Devcontainer.Namespace) + + devcontainerApp, err := devcontainer_k8s_agent_module.GetDevcontainer(&apiRequestContext, k8sClient, opts) + if err != nil { + log.Error("Get_IDE_TerminalURL: 获取 DevcontainerApp 失败: %v", err) + return "", err + } + + // 使用 NodePort 作为 SSH 端口 + port = fmt.Sprintf("%d", devcontainerApp.Status.NodePortAssigned) + log.Info("Get_IDE_TerminalURL: K8s 环境使用 NodePort %s 作为 SSH 端口", port) + + case setting.DOCKER: + // 原有 Docker 处理逻辑 + defalut_ctx := context.Background() + cli, err := docker.CreateDockerClient(&defalut_ctx) + if err != nil { + return "", err + } + defer cli.Close() + + containerID, err := docker.GetContainerID(cli, devcontainer.DevContainerName) + if err != nil { + return "", err + } + + mappedPort, err := docker.GetMappedPort(cli, containerID, "22") + if err != nil { + return "", err + } + port = mappedPort + + default: + return "", fmt.Errorf("不支持的 DevContainer Agent 类型: %s", setting.Devcontainer.Agent) + } + + // 构建并返回 URL return "://mengning.devstar/" + "openProject?host=" + devcontainer.DevContainerHost + "&port=" + port + @@ -495,7 +556,6 @@ func Get_IDE_TerminalURL(ctx *gitea_context.Context, devcontainer *RepoDevContai "&path=" + devcontainer.DevContainerWorkDir + "&access_token=" + access_token + "&devstar_username=" + devcontainer.RepoOwnerName, nil - } func AddPublicKeyToAllRunningDevContainer(ctx context.Context, user *user_model.User, publicKey string) error { diff --git a/services/devcontainer/k8s_agent.go b/services/devcontainer/k8s_agent.go index ea95a3fc23..dbf2f22fb0 100644 --- a/services/devcontainer/k8s_agent.go +++ b/services/devcontainer/k8s_agent.go @@ -3,6 +3,7 @@ package devcontainer import ( "context" "fmt" + "strings" "time" "code.gitea.io/gitea/models/db" @@ -16,9 +17,14 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/devcontainer/errors" "code.gitea.io/gitea/services/devstar_cloud_provider" + networkingv1 "k8s.io/api/networking/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" ) var k8sGroupVersionResource = schema.GroupVersionResource{ @@ -105,6 +111,32 @@ func AssignDevcontainerDeletion2K8sOperator(ctx *context.Context, devcontainersL return err } + // 获取标准 Kubernetes 客户端，用于删除 Ingress + stdClient, err := getStandardKubernetesClient() + if err != nil { + log.Warn("AssignDevcontainerDeletion2K8sOperator: 获取标准 K8s 客户端失败: %v", err) + // 继续执行，不阻止主流程 + } else { + // 先删除与 DevContainer 相关的 Ingress 资源 + for _, devcontainer := range *devcontainersList { + ingressName := fmt.Sprintf("%s-ttyd-ingress", devcontainer.Name) + log.Info("AssignDevcontainerDeletion2K8sOperator: 删除 Ingress %s", ingressName) + + err := stdClient.NetworkingV1().Ingresses(setting.Devcontainer.Namespace).Delete(*ctx, ingressName, metav1.DeleteOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + // Ingress 已经不存在，视为正常情况 + log.Info("AssignDevcontainerDeletion2K8sOperator: Ingress %s 不存在，跳过删除", ingressName) + } else { + log.Warn("AssignDevcontainerDeletion2K8sOperator: 删除 Ingress %s 失败: %v", ingressName, err) + // 继续执行，不阻止主流程 + } + } else { + log.Info("AssignDevcontainerDeletion2K8sOperator: 成功删除 Ingress %s", ingressName) + } + } + } + // 2. 调用 modules 层 k8s Agent，执行删除资源 opts := &devcontainer_dto.DeleteDevcontainerOptions{ DeleteOptions: metav1.DeleteOptions{}, @@ -120,11 +152,20 @@ func AssignDevcontainerDeletion2K8sOperator(ctx *context.Context, devcontainersL for _, devcontainer := range *devcontainersList { opts.Name = devcontainer.Name _ = devcontainer_k8s_agent_module.DeleteDevcontainer(ctx, client, opts) + + // 删除SSH端口的NAT规则 devcontainerNatRuleDescription := setting.DEVCONTAINER_CLOUD_NAT_RULE_DESCRIPTION_PREFIX + devcontainer.Name err := devstar_cloud_provider.DeleteNATRulePort(devcontainerNatRuleDescription) if err != nil { log.Warn("[Cloud NAT DELETION Error]: " + err.Error()) } + + // 删除ttyd端口的NAT规则 + ttydNatRuleDescription := setting.DEVCONTAINER_CLOUD_NAT_RULE_DESCRIPTION_PREFIX + devcontainer.Name + "-ttyd" + err = devstar_cloud_provider.DeleteNATRulePort(ttydNatRuleDescription) + if err != nil { + log.Warn("[Cloud ttyd NAT DELETION Error]: " + err.Error()) + } } return nil } @@ -317,11 +358,43 @@ func AssignDevcontainerCreation2K8sOperator(ctx *context.Context, newDevContaine } else { log.Info("AssignDevcontainerCreation2K8sOperator: NAT port mapping created successfully") } + + // 为ttyd端口创建NAT映射 - 查找extraPorts中的ttyd端口 + for _, portInfo := range devcontainerInCluster.Status.ExtraPortsAssigned { + if portInfo.Name == "ttyd" || strings.Contains(portInfo.Name, "ttyd") || + portInfo.Name == "port-7681" || portInfo.ContainerPort == 7681 { + // 找到ttyd端口，创建NAT映射 + ttydPrivatePort := uint64(portInfo.NodePort) + ttydPublicPort := ttydPrivatePort + ttydNatRuleDescription := setting.DEVCONTAINER_CLOUD_NAT_RULE_DESCRIPTION_PREFIX + newDevContainer.Name + "-ttyd" + + log.Info("AssignDevcontainerCreation2K8sOperator: Creating NAT port mapping for ttyd: private=%d, public=%d, desc=%s", + ttydPrivatePort, ttydPublicPort, ttydNatRuleDescription) + + err = devstar_cloud_provider.CreateNATRulePort(ttydPrivatePort, ttydPublicPort, ttydNatRuleDescription) + if err != nil { + log.Error("AssignDevcontainerCreation2K8sOperator: Failed to create NAT port mapping for ttyd: %v", err) + } else { + log.Info("AssignDevcontainerCreation2K8sOperator: NAT port mapping for ttyd created successfully") + } + break + } + } } log.Info("DevContainer created in cluster - Name: %s, NodePort: %d", devcontainerInCluster.Name, devcontainerInCluster.Status.NodePortAssigned) + + // 为 ttyd 服务创建 Ingress + err = createDevContainerTTYDIngress(ctx, devcontainerInCluster.Name) + if err != nil { + log.Warn("AssignDevcontainerCreation2K8sOperator: 创建 ttyd Ingress 失败: %v", err) + // 不阻止主流程，只记录警告 + } else { + log.Info("AssignDevcontainerCreation2K8sOperator: 创建 ttyd Ingress 成功") + } + // 4. 层层返回 nil，自动提交数据库事务，完成 DevContainer 创建 return nil } @@ -490,11 +563,29 @@ func updateNodePortAsync(containerName string, namespace string, userId, repoId log.Info("updateNodePortAsync: NAT port mapping created successfully") } - // 记录 ttyd 端口信息到日志 + // 记录 ttyd 端口信息到日志并创建NAT映射 if len(devcontainer.Status.ExtraPortsAssigned) > 0 { for _, portInfo := range devcontainer.Status.ExtraPortsAssigned { log.Info("Found extra port for %s: name=%s, nodePort=%d, containerPort=%d", containerName, portInfo.Name, portInfo.NodePort, portInfo.ContainerPort) + + // 为ttyd端口创建NAT映射 + if portInfo.Name == "ttyd" || strings.Contains(portInfo.Name, "ttyd") || + portInfo.Name == "port-7681" || portInfo.ContainerPort == 7681 { + ttydPrivatePort := uint64(portInfo.NodePort) + ttydPublicPort := ttydPrivatePort + ttydNatRuleDescription := setting.DEVCONTAINER_CLOUD_NAT_RULE_DESCRIPTION_PREFIX + containerName + "-ttyd" + + log.Info("updateNodePortAsync: Creating NAT port mapping for ttyd: private=%d, public=%d, desc=%s", + ttydPrivatePort, ttydPublicPort, ttydNatRuleDescription) + + err := devstar_cloud_provider.CreateNATRulePort(ttydPrivatePort, ttydPublicPort, ttydNatRuleDescription) + if err != nil { + log.Error("updateNodePortAsync: Failed to create NAT port mapping for ttyd: %v", err) + } else { + log.Info("updateNodePortAsync: NAT port mapping for ttyd created successfully") + } + } } } @@ -523,3 +614,121 @@ func updateNodePortAsync(containerName string, namespace string, userId, repoId log.Warn("updateNodePortAsync: Failed to retrieve real NodePort after multiple attempts") } + +// 创建 DevContainer ttyd 服务的 Ingress 资源 +func createDevContainerTTYDIngress(ctx *context.Context, devcontainerName string) error { + log.Info("createDevContainerTTYDIngress: 开始为 %s 创建 ttyd Ingress", devcontainerName) + + // 获取标准 Kubernetes 客户端 + stdClient, err := getStandardKubernetesClient() + if err != nil { + log.Error("createDevContainerTTYDIngress: 获取 K8s 客户端失败: %v", err) + return err + } + + // 从配置中读取域名 + cfg, err := setting.NewConfigProviderFromFile(setting.CustomConf) + if err != nil { + log.Error("createDevContainerTTYDIngress: 加载配置文件失败: %v", err) + return err + } + + domain := cfg.Section("server").Key("DOMAIN").Value() + + // 从容器名称中提取用户名和仓库名 + parts := strings.Split(devcontainerName, "-") + var username, repoName string + if len(parts) >= 2 { + username = parts[0] + repoName = parts[1] + } else { + username = "unknown" + repoName = "unknown" + } + + // 构建 PATH 和 Ingress 配置 + path := fmt.Sprintf("/%s/%s/dev-container-webterminal", username, repoName) + ingressName := fmt.Sprintf("%s-ttyd-ingress", devcontainerName) + serviceName := fmt.Sprintf("%s-svc", devcontainerName) + + // 创建 Ingress 资源 + pathType := networkingv1.PathTypePrefix + ingressClassName := "nginx" + + ingress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressName, + Namespace: setting.Devcontainer.Namespace, + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/proxy-send-timeout": "3600", + "nginx.ingress.kubernetes.io/proxy-read-timeout": "3600", + "nginx.ingress.kubernetes.io/websocket-services": serviceName, + "nginx.ingress.kubernetes.io/proxy-http-version": "1.1", + "nginx.ingress.kubernetes.io/rewrite-target": "/", + "nginx.ingress.kubernetes.io/configuration-snippet": ` + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-For $remote_addr; + `, + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: &ingressClassName, + Rules: []networkingv1.IngressRule{ + { + Host: domain, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: path, + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: serviceName, + Port: networkingv1.ServiceBackendPort{ + Number: 7681, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + // 创建 Ingress 资源 + _, err = stdClient.NetworkingV1().Ingresses(setting.Devcontainer.Namespace).Create(*ctx, ingress, metav1.CreateOptions{}) + if err != nil { + log.Error("createDevContainerTTYDIngress: 创建 Ingress 失败: %v", err) + return err + } + + log.Info("createDevContainerTTYDIngress: 成功创建 Ingress %s，ttyd 可通过 https://%s%s 访问", ingressName, domain, path) + return nil +} + +// 获取标准 Kubernetes 客户端 +func getStandardKubernetesClient() (*kubernetes.Clientset, error) { + // 使用与 GetKubernetesClient 相同的逻辑获取配置 + config, err := clientcmd.BuildConfigFromFlags("", clientcmd.RecommendedHomeFile) + if err != nil { + // 如果集群外配置失败，尝试集群内配置 + log.Warn("Failed to obtain Kubernetes config outside of cluster: %v", err) + config, err = rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("获取 K8s 配置失败 (集群内外均失败): %v", err) + } + } + + // 创建标准客户端 + stdClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("创建标准 K8s 客户端失败: %v", err) + } + + return stdClient, nil +}