package devcontainer import ( "context" "fmt" "io" "io/ioutil" "os" "path/filepath" "regexp" "strings" "time" "code.gitea.io/gitea/models/db" devcontainer_model "code.gitea.io/gitea/models/devcontainer" devcontainer_models_errors "code.gitea.io/gitea/models/devcontainer/errors" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/docker" git_module "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" gitea_context "code.gitea.io/gitea/services/context" devcontainer_service_errors "code.gitea.io/gitea/services/devcontainer/errors" "code.gitea.io/gitea/services/devstar_ssh_key_pair/api_service" "github.com/google/uuid" "xorm.io/builder" ) // 封装查询 DevContainer 实时信息,与具体 agent 无关,返回前端 type OpenDevcontainerAbstractAgent struct { NodePortAssigned uint16 } // OpenDevcontainerService 获取 DevContainer 连接信息,抽象方法,适配多种 DevContainer Agent func OpenDevcontainerService(ctx *gitea_context.Context, opts *OpenDevcontainerAppDispatcherOptions) (*OpenDevcontainerAbstractAgent, error) { // 0. 检查参数 if ctx == nil || opts == nil || len(opts.Name) == 0 { return nil, devcontainer_service_errors.ErrIllegalParams{ FieldNameList: []string{"ctx", "opts.Name"}, } } // 1. 检查 DevContainer 功能是否开启 if setting.Devcontainer.Enabled == false { return nil, devcontainer_service_errors.ErrOperateDevcontainer{ Action: "check availability of DevStar DevContainer", Message: "DevContainer is turned off globally", } } // 2. 根据 DevContainer Agent 类型分发任务 apiRequestContext := ctx.Req.Context() openDevcontainerAbstractAgentVO := &OpenDevcontainerAbstractAgent{} switch setting.Devcontainer.Agent { case setting.KUBERNETES: devcontainerApp, err := AssignDevcontainerGetting2K8sOperator(&apiRequestContext, opts) if err != nil { return nil, devcontainer_service_errors.ErrOperateDevcontainer{ Action: "Open DevContainer in k8s", Message: err.Error(), } } openDevcontainerAbstractAgentVO.NodePortAssigned = devcontainerApp.Status.NodePortAssigned case setting.DOCKER: port, err := GetDevcontainer(&apiRequestContext, opts) log.Info("port %d", port) if err != nil { return nil, devcontainer_service_errors.ErrOperateDevcontainer{ Action: "Open DevContainer in docker", Message: err.Error(), } } openDevcontainerAbstractAgentVO.NodePortAssigned = port default: return nil, devcontainer_service_errors.ErrOperateDevcontainer{ Action: "Open DevContainer", Message: "No Valid DevContainer Agent Found", } } // 3. 封装返回结果 return openDevcontainerAbstractAgentVO, nil } // GetRepoDevcontainerDetails 获取仓库对应 DevContainer 信息 func GetRepoDevcontainerDetails(ctx context.Context, opts *RepoDevcontainerOptions) (RepoDevContainer, error) { // 0. 构造异常返回时候的空数据 resultRepoDevcontainerDetail := RepoDevContainer{} // 1. 检查参数是否有效 if opts == nil || opts.Actor == nil || opts.Repository == nil { return resultRepoDevcontainerDetail, devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: "construct query condition for devContainer user list", Message: "invalid search condition", } } // 2. 查询数据库 /* SELECT devstar_devcontainer.id AS devcontainer_id, devstar_devcontainer.name AS devcontainer_name, devstar_devcontainer.devcontainer_host AS devcontainer_host, devstar_devcontainer.devcontainer_port AS devcontainer_port, devstar_devcontainer.devcontainer_username AS devcontainer_username, devstar_devcontainer.devcontainer_work_dir AS devcontainer_work_dir, devstar_devcontainer.repo_id AS repo_id, repository.name AS repo_name, repository.owner_name AS repo_owner_name, repository.description AS repo_description, CONCAT('/', repository.owner_name, '/', repository.name) AS repo_link FROM devstar_devcontainer INNER JOIN repository on devstar_devcontainer.repo_id = repository.id WHERE devstar_devcontainer.user_id = #{opts.Actor.ID} AND devstar_devcontainer.repo_id = #{opts.Repository.ID}; */ _, err := db.GetEngine(ctx). Table("devstar_devcontainer"). Select(""+ "devstar_devcontainer.id AS devcontainer_id,"+ "devstar_devcontainer.name AS devcontainer_name,"+ "devstar_devcontainer.devcontainer_host AS devcontainer_host,"+ "devstar_devcontainer.devcontainer_port AS devcontainer_port,"+ "devstar_devcontainer.devcontainer_username AS devcontainer_username,"+ "devstar_devcontainer.devcontainer_work_dir AS devcontainer_work_dir,"+ "devstar_devcontainer.repo_id AS repo_id,"+ "repository.name AS repo_name,"+ "repository.owner_name AS repo_owner_name,"+ "repository.description AS repo_description,"+ "CONCAT('/', repository.owner_name, '/', repository.name) AS repo_link"). Join("INNER", "repository", "devstar_devcontainer.repo_id = repository.id"). Where("devstar_devcontainer.user_id = ? AND devstar_devcontainer.repo_id = ?", opts.Actor.ID, opts.Repository.ID). Get(&resultRepoDevcontainerDetail) // 3. 返回 if err != nil { return resultRepoDevcontainerDetail, devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("query devcontainer with repo '%v' and username '%v'", opts.Repository.Name, opts.Actor.Name), Message: err.Error(), } } return resultRepoDevcontainerDetail, nil } // CreateRepoDevcontainer 创建 DevContainer /* 必要假设:前置中间件已完成检查,确保数据有效: - 当前用户为已登录用户 - 当前用户拥有 repo code写入权限 - 数据库此前不存在 该用户在该repo创建的 Dev Container */ func CreateRepoDevcontainer(ctx context.Context, opts *CreateRepoDevcontainerOptions) error { username := opts.Actor.Name repoName := opts.Repository.Name // unixTimestamp is the number of seconds elapsed since January 1, 1970 UTC. unixTimestamp := time.Now().Unix() newDevcontainer := &CreateDevcontainerDTO{ DevstarDevcontainer: devcontainer_model.DevstarDevcontainer{ Name: getSanitizedDevcontainerName(username, repoName), DevcontainerHost: setting.Devcontainer.Host, DevcontainerUsername: "root", DevcontainerWorkDir: "/data/workspace", RepoId: opts.Repository.ID, UserId: opts.Actor.ID, CreatedUnix: unixTimestamp, UpdatedUnix: unixTimestamp, }, Image: GetDefaultDevcontainerImageFromRepoDevcontainerJSON(ctx, opts.Repository), GitRepositoryURL: strings.TrimSuffix(setting.AppURL, "/") + opts.Repository.Link(), } // 在数据库事务中创建 Dev Container 分配资源,出错时自动回滚相对应数据库字段,保证数据一致 dbTransactionErr := db.WithTx(ctx, func(ctx context.Context) error { var err error // 0. 查询数据库,收集用户 SSH 公钥,合并用户临时填入SSH公钥 // (若用户合计 SSH公钥个数为0,拒绝创建DevContainer) /** SELECT content FROM public_key where owner_id = #{opts.Actor.ID} */ var userSSHPublicKeyList []string err = db.GetEngine(ctx). Table("public_key"). Select("content"). Where("owner_id = ?", opts.Actor.ID). Find(&userSSHPublicKeyList) if err != nil { return devcontainer_service_errors.ErrOperateDevcontainer{ Action: fmt.Sprintf("query SSH Public Key List for User %s", opts.Actor.Name), Message: err.Error(), } } newDevcontainer.SSHPublicKeyList = append(userSSHPublicKeyList, opts.SSHPublicKeyList...) devstarPublicKey := getDevStarPublicKey() if devstarPublicKey == "" { return devcontainer_service_errors.ErrOperateDevcontainer{ Action: fmt.Sprintf("devstar SSH Public Key Error "), Message: err.Error(), } } newDevcontainer.SSHPublicKeyList = append(newDevcontainer.SSHPublicKeyList) // if len(userSSHPublicKeyList) <= 0 { // // API没提供临时SSH公钥,用户后台也没有永久SSH公钥,直接结束并回滚事务 // return devcontainer_service_errors.ErrOperateDevcontainer{ // Action: "Check SSH Public Key List", // Message: "禁止创建无法连通的DevContainer:用户未提供 SSH 公钥,请先使用API临时创建SSH密钥对、或在Web端手动添加SSH公钥", // } // } // 1. 调用 k8s Operator Agent,创建 DevContainer 资源,同时更新k8s调度器分配的 NodePort err = claimDevcontainerResource(&ctx, newDevcontainer) if err != nil { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("claim resource for Dev Container %v", newDevcontainer), Message: err.Error(), } } // 2. 根据分配的 NodePort 更新数据库字段 rowsAffect, err := db.GetEngine(ctx). Table("devstar_devcontainer"). Insert(newDevcontainer.DevstarDevcontainer) if err != nil { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("insert new DevContainer for user '%s' in repo '%s'", username, repoName), Message: err.Error(), } } else if rowsAffect == 0 { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("insert new DevContainer for user '%s' in repo '%s'", username, repoName), Message: "expected 1 row to be inserted, but got 0", } } return nil }) return dbTransactionErr } func getDevStarPublicKey() string { // 获取当前用户的主目录 homeDir, err := os.UserHomeDir() if err != nil { log.Info("Failed to get home directory: %s", err) } // 构建公钥文件的路径 publicKeyPath := filepath.Join(homeDir, ".ssh", "id_rsa_devstar.pub") privateKeyPath := filepath.Join(homeDir, ".ssh", "id_rsa_devstar") if !fileExists(publicKeyPath) || !fileExists(privateKeyPath) { err, key := api_service.GenerateNewRSASSHSessionKeyPair() if err != nil { log.Info("无法创建密钥:", err) return "" } // 确保~/.ssh目录存在 sshDir := filepath.Join(homeDir, ".ssh") if err := os.MkdirAll(sshDir, 0700); err != nil { log.Info("无法创建~/.ssh目录:", err) return "" } // 创建密钥文件 if err := ioutil.WriteFile(privateKeyPath, []byte(key.PrivateKeyPEM), 0600); err != nil { log.Info("无法写入私钥文件:", err) return "" } if err := ioutil.WriteFile(publicKeyPath, []byte(key.PublicKeySsh), 0600); err != nil { log.Info("无法写入公钥文件:", err) return "" } } // 读取公钥文件内容 publicKeyBytes, err := ioutil.ReadFile(publicKeyPath) if err != nil { log.Info("Failed to read public key file: %s", err) return "" } // 将文件内容转换为字符串 return string(publicKeyBytes) } func fileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { return false } return !info.IsDir() } func GetWebTerminalURL(ctx context.Context, devcontainerName string) (string, error) { switch setting.Devcontainer.Agent { case setting.KUBERNETES: return "", fmt.Errorf("unsupported agent") case setting.DOCKER: cli, err := docker.CreateDockerClient(&ctx) if err != nil { return "", err } defer cli.Close() containerID, err := docker.GetContainerID(cli, devcontainerName) if err != nil { return "", err } port, err := docker.GetMappedPort(cli, containerID, "7681") if err != nil { return "", err } return "http://" + setting.Devcontainer.Host + ":" + port + "/", nil default: return "", fmt.Errorf("unknown agent") } } // DeleteRepoDevcontainer 按照 仓库 和/或 用户信息删除 DevContainer(s) func DeleteRepoDevcontainer(ctx context.Context, opts *RepoDevcontainerOptions) error { if ctx == nil || opts == nil || (opts.Actor == nil && opts.Repository == nil) { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: "construct query parameters", Message: "Invalid parameters", } } // 1. 构造查询条件 sqlDevcontainerCondition := builder.NewCond() if opts.Actor != nil { sqlDevcontainerCondition = sqlDevcontainerCondition.And(builder.Eq{"user_id": opts.Actor.ID}) } if opts.Repository != nil { sqlDevcontainerCondition = sqlDevcontainerCondition.And(builder.Eq{"repo_id": opts.Repository.ID}) } var devcontainersList []devcontainer_model.DevstarDevcontainer // 2. 开启事务:先获取 devcontainer列表,后删除 dbTransactionErr := db.WithTx(ctx, func(ctx context.Context) error { var err error // 2.1 条件查询: user_id 和/或 repo_id err = db.GetEngine(ctx). Table("devstar_devcontainer"). Where(sqlDevcontainerCondition). Find(&devcontainersList) if err != nil { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("find devcontainer(s) with condition '%v'", sqlDevcontainerCondition), Message: err.Error(), } } // 2.2 空列表,直接结束事务(由于前一个操作只是查询,所以回滚事务不会导致数据不一致问题) if len(devcontainersList) == 0 { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("find devcontainer(s) with condition '%v'", sqlDevcontainerCondition), Message: "No DevContainer found", } } // 2.3 条件删除: user_id 和/或 repo_id _, err = db.GetEngine(ctx). Table("devstar_devcontainer"). Where(sqlDevcontainerCondition). Delete() if err != nil { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: fmt.Sprintf("MARK devcontainer(s) as DELETED with condition '%v'", sqlDevcontainerCondition), Message: err.Error(), } } return nil }) if dbTransactionErr != nil { return dbTransactionErr } // 3. 后台启动一个goroutine慢慢回收 Dev Container 资源 (如果回收失败,将会产生孤儿 Dev Container,只能管理员手动识别、删除) go func() { // 注意:由于执行删除 k8s 资源 与 数据库交互和Web页面更新是异步的,因此在 goroutine 中必须重新创建 context,否则报错: // Delete "https://192.168.49.2:8443/apis/devcontainer.devstar.cn/v1/...": context canceled isolatedContextToPurgeK8sResource, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() _ = purgeDevcontainersResource(&isolatedContextToPurgeK8sResource, &devcontainersList) }() return dbTransactionErr } // getSanitizedDevcontainerName 辅助获取当前用户在当前仓库创建的 devContainer名称 // DevContainer命名规则: 用户名、仓库名各由小于15位的小写字母和数字组成,中间使用'-'分隔,后面使用'-'分隔符拼接32位UUID小写字母数字字符串 func getSanitizedDevcontainerName(username, repoName string) string { regexpNonAlphaNum := regexp.MustCompile(`[^a-zA-Z0-9]`) sanitizedUsername := regexpNonAlphaNum.ReplaceAllString(username, "") sanitizedRepoName := regexpNonAlphaNum.ReplaceAllString(repoName, "") if len(sanitizedUsername) > 15 { sanitizedUsername = strings.ToLower(sanitizedUsername[:15]) } if len(sanitizedRepoName) > 31 { sanitizedRepoName = strings.ToLower(sanitizedRepoName[:31]) } newUUID, _ := uuid.NewUUID() uuidStr := newUUID.String() uuidStr = regexpNonAlphaNum.ReplaceAllString(uuidStr, "")[:15] return fmt.Sprintf("%s-%s-%s", sanitizedUsername, sanitizedRepoName, uuidStr) } // purgeDevcontainersResource 辅助函数,用于goroutine后台执行,回收DevContainer资源 func purgeDevcontainersResource(ctx *context.Context, devcontainersList *[]devcontainer_model.DevstarDevcontainer) error { // 1. 检查 DevContainer 功能是否启用,若禁用,则直接结束,不会真正执行删除操作 if !setting.Devcontainer.Enabled { // 如果用户设置禁用 DevContainer,无法删除资源,会直接忽略,而数据库相关记录会继续清空、不会发生回滚 log.Warn("Orphan DevContainers in namespace `%s` left undeleted: %v", setting.Devcontainer.Namespace, devcontainersList) return nil } // 2. 根据配置文件中指定的 DevContainer Agent 派遣创建任务 switch setting.Devcontainer.Agent { case setting.KUBERNETES: return AssignDevcontainerDeletion2K8sOperator(ctx, devcontainersList) case setting.DOCKER: return DeleteDevcontainer(ctx, devcontainersList) default: // 未知 Agent,直接报错 return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: "dispatch DevContainer deletion", Message: fmt.Sprintf("unknown DevContainer agent `%s`, please check your config files", setting.Devcontainer.Agent), } } } // claimDevcontainerResource 分发创建 DevContainer 任务到配置文件指定的执行器 func claimDevcontainerResource(ctx *context.Context, newDevContainer *CreateDevcontainerDTO) error { // 1. 检查 DevContainer 功能是否启用,若禁用,则直接结束 if !setting.Devcontainer.Enabled { return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: "Check for DevContainer functionality switch", Message: "DevContainer is disabled globally, please check your configuration files", } } // 2. 根据配置文件中指定的 DevContainer Agent 派遣创建任务 switch setting.Devcontainer.Agent { case setting.KUBERNETES, "k8s": // k8s Operator return AssignDevcontainerCreation2K8sOperator(ctx, newDevContainer) case setting.DOCKER: return CreateDevcontainer(ctx, newDevContainer) default: // 未知 Agent,直接报错 return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{ Action: "dispatch DevContainer creation", Message: fmt.Sprintf("unknown DevContainer agent `%s`, please check your config files", setting.Devcontainer.Agent), } } } // GetDefaultDevcontainerImageFromRepoDevcontainerJSON 从 .devcontainer/devcontainer.json 中获取 devContainer image func GetDefaultDevcontainerImageFromRepoDevcontainerJSON(ctx context.Context, repo *repo_model.Repository) string { // 1. 获取默认分支名 branchName := repo.DefaultBranch if len(branchName) == 0 { branchName = setting.Devcontainer.DefaultGitBranchName } // 2. 打开默认分支 gitRepoEntity, err := git_module.OpenRepository(ctx, repo.RepoPath()) if err != nil { log.Error("Failed to open repository %s: %v", repo.RepoPath(), err) return setting.Devcontainer.DefaultDevcontainerImageName } defer func(gitRepoEntity *git_module.Repository) { _ = gitRepoEntity.Close() }(gitRepoEntity) // 3. 获取分支名称 commit, err := gitRepoEntity.GetBranchCommit(branchName) if err != nil { return setting.Devcontainer.DefaultDevcontainerImageName } // 4. 读取 .devcontainer/devcontainer.json 文件 const maxDevcontainerJSONSize = 10 * 1024 * 1024 // 设置最大允许的文件大小 10MB devcontainerJSONContent, err := commit.GetFileContent(".devcontainer/devcontainer.json", maxDevcontainerJSONSize) if err != nil { log.Error("Failed to get .devcontainer/devcontainer.json file: %v", err) return setting.Devcontainer.DefaultDevcontainerImageName } // 5. 移除注释 cleanedContent, err := removeComments(devcontainerJSONContent) if err != nil { log.Error("Failed to remove comments from .devcontainer/devcontainer.json: %v", err) return setting.Devcontainer.DefaultDevcontainerImageName } // 5. 解析 JSON devContainerJSON, err := devcontainer_model.Unmarshal(cleanedContent) if err != nil { log.Error("Failed to unmarshal .devcontainer/devcontainer.json: %v", err) return setting.Devcontainer.DefaultDevcontainerImageName } // 6. 解析并返回 if len(devContainerJSON.Image) == 0 { return setting.Devcontainer.DefaultDevcontainerImageName } return devContainerJSON.Image } func GetDevcontainerJSONContent(ctx *gitea_context.Context) (string, error) { entry, err := ctx.Repo.Commit.GetTreeEntryByPath(".devcontainer/devcontainer.json") if err != nil { log.Info("Repo.Commit.GetTreeEntryByPath %v", err.Error()) return "", err } // No way to edit a directory online. if entry.IsDir() { ctx.NotFound("entry.IsDir", nil) return "", fmt.Errorf(".devcontainer/devcontainer.json entry.IsDir") } blob := entry.Blob() if blob.Size() >= setting.UI.MaxDisplayFileSize { ctx.NotFound("blob.Size", err) return "", fmt.Errorf(".devcontainer/devcontainer.json blob.Size overflow") } dataRc, err := blob.DataAsync() if err != nil { ctx.NotFound("blob.Data", err) return "", err } defer dataRc.Close() buf := make([]byte, 1024) n, _ := util.ReadAtMost(dataRc, buf) buf = buf[:n] // Only some file types are editable online as text. if !typesniffer.DetectContentType(buf).IsRepresentableAsText() { ctx.NotFound("typesniffer.IsRepresentableAsText", nil) return "", fmt.Errorf("typesniffer.IsRepresentableAsText") } d, _ := io.ReadAll(dataRc) buf = append(buf, d...) if content, err := charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true}); err != nil { log.Error("ToUTF8: %v", err) return string(buf), nil } else { return content, nil } } // 移除 JSON 文件中的注释 func removeComments(data string) (string, error) { // 移除单行注释 // ... re := regexp.MustCompile(`//.*`) data = re.ReplaceAllString(data, "") // 移除多行注释 /* ... */ re = regexp.MustCompile(`/\*.*?\*/`) data = re.ReplaceAllString(data, "") return data, nil }