e6d1dbb381
* 合并devcontainer web相关的源文件，简化目录结构 * devcontainer、ssh_key_pair和devcontainer.cloud * fixed bug：创建容器时Host为localhost时创建失败的问题 * 删除了死代码，更新了一些命名（主要是去掉devstar字符串） * 常量名全大写 * devcontainer HOST改为用户设置的域名或IP * 安装时如没有配置devcontainer则默认设置为docker方式 * 直接使用kubernetes和docker简化代码提高可读性 * 去除services/devstar_devcontainer文件夹名中不必要的devstar字符串 * 去除services/devstar_devcontainer文件夹名中不必要的devstar字符串 * 文件名中去掉不必要的devstar字符串 * 变量名中删除不必要的Devstar字符串 * Merge branch 'dev' into feature-websshAndUpdateImage * change pages style * change Structure * fix bug * websshAndUpdateImage
425 lines
18 KiB
Go
425 lines
18 KiB
Go
package devcontainer
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io/ioutil"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"strings"
|
||
"time"
|
||
|
||
"code.gitea.io/gitea/modules/docker"
|
||
|
||
"code.gitea.io/gitea/models/db"
|
||
devcontainer_models "code.gitea.io/gitea/models/devstar_devcontainer"
|
||
devcontainer_models_errors "code.gitea.io/gitea/models/devstar_devcontainer/errors"
|
||
repo_model "code.gitea.io/gitea/models/repo"
|
||
git_module "code.gitea.io/gitea/modules/git"
|
||
"code.gitea.io/gitea/modules/log"
|
||
"code.gitea.io/gitea/modules/setting"
|
||
DevcontainersVO "code.gitea.io/gitea/routers/api/devcontainer/vo"
|
||
devcontainer_service_dto "code.gitea.io/gitea/services/devcontainer/dto"
|
||
devcontainer_service_errors "code.gitea.io/gitea/services/devcontainer/errors"
|
||
devcontainer_k8s_agent_service "code.gitea.io/gitea/services/devcontainer/k8s_agent"
|
||
"code.gitea.io/gitea/services/devstar_ssh_key_pair/api_service"
|
||
"github.com/google/uuid"
|
||
"xorm.io/builder"
|
||
)
|
||
|
||
// GetRepoDevcontainerDetails 获取仓库对应 DevContainer 信息
|
||
func GetRepoDevcontainerDetails(ctx context.Context, opts *DevcontainersVO.RepoDevcontainerOptions) (DevcontainersVO.RepoDevContainerVO, error) {
|
||
|
||
// 0. 构造异常返回时候的空数据
|
||
resultRepoDevcontainerDetail := DevcontainersVO.RepoDevContainerVO{}
|
||
|
||
// 1. 检查参数是否有效
|
||
if opts == nil || opts.Actor == nil || opts.Repository == nil {
|
||
return resultRepoDevcontainerDetail, devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: "construct query condition for devContainer user list",
|
||
Message: "invalid search condition",
|
||
}
|
||
}
|
||
|
||
// 2. 查询数据库
|
||
/*
|
||
SELECT
|
||
devstar_devcontainer.id AS devcontainer_id,
|
||
devstar_devcontainer.name AS devcontainer_name,
|
||
devstar_devcontainer.devcontainer_host AS devcontainer_host,
|
||
devstar_devcontainer.devcontainer_port AS devcontainer_port,
|
||
devstar_devcontainer.devcontainer_username AS devcontainer_username,
|
||
devstar_devcontainer.devcontainer_work_dir AS devcontainer_work_dir,
|
||
devstar_devcontainer.repo_id AS repo_id,
|
||
repository.name AS repo_name,
|
||
repository.owner_name AS repo_owner_name,
|
||
repository.description AS repo_description,
|
||
CONCAT('/', repository.owner_name, '/', repository.name) AS repo_link
|
||
FROM devstar_devcontainer
|
||
INNER JOIN repository on devstar_devcontainer.repo_id = repository.id
|
||
WHERE
|
||
devstar_devcontainer.user_id = #{opts.Actor.ID}
|
||
AND
|
||
devstar_devcontainer.repo_id = #{opts.Repository.ID};
|
||
*/
|
||
_, err := db.GetEngine(ctx).
|
||
Table("devstar_devcontainer").
|
||
Select(""+
|
||
"devstar_devcontainer.id AS devcontainer_id,"+
|
||
"devstar_devcontainer.name AS devcontainer_name,"+
|
||
"devstar_devcontainer.devcontainer_host AS devcontainer_host,"+
|
||
"devstar_devcontainer.devcontainer_port AS devcontainer_port,"+
|
||
"devstar_devcontainer.devcontainer_username AS devcontainer_username,"+
|
||
"devstar_devcontainer.devcontainer_work_dir AS devcontainer_work_dir,"+
|
||
"devstar_devcontainer.repo_id AS repo_id,"+
|
||
"repository.name AS repo_name,"+
|
||
"repository.owner_name AS repo_owner_name,"+
|
||
"repository.description AS repo_description,"+
|
||
"CONCAT('/', repository.owner_name, '/', repository.name) AS repo_link").
|
||
Join("INNER", "repository", "devstar_devcontainer.repo_id = repository.id").
|
||
Where("devstar_devcontainer.user_id = ? AND devstar_devcontainer.repo_id = ?", opts.Actor.ID, opts.Repository.ID).
|
||
Get(&resultRepoDevcontainerDetail)
|
||
|
||
// 3. 返回
|
||
if err != nil {
|
||
return resultRepoDevcontainerDetail, devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("query devcontainer with repo '%v' and username '%v'", opts.Repository.Name, opts.Actor.Name),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
return resultRepoDevcontainerDetail, nil
|
||
}
|
||
|
||
// CreateRepoDevcontainer 创建 DevContainer
|
||
/*
|
||
必要假设：前置中间件已完成检查，确保数据有效：
|
||
- 当前用户为已登录用户
|
||
- 当前用户拥有 repo code写入权限
|
||
- 数据库此前不存在 该用户在该repo创建的 Dev Container
|
||
*/
|
||
func CreateRepoDevcontainer(ctx context.Context, opts *DevcontainersVO.CreateRepoDevcontainerOptions) error {
|
||
username := opts.Actor.Name
|
||
repoName := opts.Repository.Name
|
||
|
||
// unixTimestamp is the number of seconds elapsed since January 1, 1970 UTC.
|
||
unixTimestamp := time.Now().Unix()
|
||
|
||
newDevcontainer := &devcontainer_service_dto.CreateDevcontainerDTO{
|
||
DevstarDevcontainer: devcontainer_models.DevstarDevcontainer{
|
||
Name: getSanitizedDevcontainerName(username, repoName),
|
||
DevcontainerHost: setting.Devcontainer.Host,
|
||
DevcontainerUsername: "root",
|
||
DevcontainerWorkDir: "/data/workspace",
|
||
RepoId: opts.Repository.ID,
|
||
UserId: opts.Actor.ID,
|
||
CreatedUnix: unixTimestamp,
|
||
UpdatedUnix: unixTimestamp,
|
||
},
|
||
Image: GetDefaultDevcontainerImageFromRepoDevcontainerJSON(ctx, opts.Repository),
|
||
GitRepositoryURL: strings.TrimSuffix(setting.AppURL, "/") + opts.Repository.Link(),
|
||
}
|
||
|
||
// 在数据库事务中创建 Dev Container 分配资源，出错时自动回滚相对应数据库字段，保证数据一致
|
||
dbTransactionErr := db.WithTx(ctx, func(ctx context.Context) error {
|
||
var err error
|
||
// 0. 查询数据库，收集用户 SSH 公钥，合并用户临时填入SSH公钥
|
||
// （若用户合计 SSH公钥个数为0，拒绝创建DevContainer）
|
||
/**
|
||
SELECT content FROM public_key where owner_id = #{opts.Actor.ID}
|
||
*/
|
||
var userSSHPublicKeyList []string
|
||
err = db.GetEngine(ctx).
|
||
Table("public_key").
|
||
Select("content").
|
||
Where("owner_id = ?", opts.Actor.ID).
|
||
Find(&userSSHPublicKeyList)
|
||
if err != nil {
|
||
return devcontainer_service_errors.ErrOperateDevcontainer{
|
||
Action: fmt.Sprintf("query SSH Public Key List for User %s", opts.Actor.Name),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
newDevcontainer.SSHPublicKeyList = append(userSSHPublicKeyList, opts.SSHPublicKeyList...)
|
||
devstarPublicKey := getDevStarPublicKey()
|
||
if devstarPublicKey == "" {
|
||
return devcontainer_service_errors.ErrOperateDevcontainer{
|
||
Action: fmt.Sprintf("devstar SSH Public Key Error "),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
newDevcontainer.SSHPublicKeyList = append(newDevcontainer.SSHPublicKeyList)
|
||
// if len(userSSHPublicKeyList) <= 0 {
|
||
// // API没提供临时SSH公钥，用户后台也没有永久SSH公钥，直接结束并回滚事务
|
||
// return devcontainer_service_errors.ErrOperateDevcontainer{
|
||
// Action: "Check SSH Public Key List",
|
||
// Message: "禁止创建无法连通的DevContainer：用户未提供 SSH 公钥，请先使用API临时创建SSH密钥对、或在Web端手动添加SSH公钥",
|
||
// }
|
||
// }
|
||
|
||
// 1. 调用 k8s Operator Agent，创建 DevContainer 资源，同时更新k8s调度器分配的 NodePort
|
||
err = claimDevcontainerResource(&ctx, newDevcontainer)
|
||
if err != nil {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("claim resource for Dev Container %v", newDevcontainer),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
// 2. 根据分配的 NodePort 更新数据库字段
|
||
rowsAffect, err := db.GetEngine(ctx).
|
||
Table("devstar_devcontainer").
|
||
Insert(newDevcontainer.DevstarDevcontainer)
|
||
if err != nil {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("insert new DevContainer for user '%s' in repo '%s'", username, repoName),
|
||
Message: err.Error(),
|
||
}
|
||
} else if rowsAffect == 0 {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("insert new DevContainer for user '%s' in repo '%s'", username, repoName),
|
||
Message: "expected 1 row to be inserted, but got 0",
|
||
}
|
||
}
|
||
return nil
|
||
})
|
||
|
||
return dbTransactionErr
|
||
}
|
||
func getDevStarPublicKey() string {
|
||
// 获取当前用户的主目录
|
||
homeDir, err := os.UserHomeDir()
|
||
if err != nil {
|
||
log.Info("Failed to get home directory: %s", err)
|
||
}
|
||
|
||
// 构建公钥文件的路径
|
||
publicKeyPath := filepath.Join(homeDir, ".ssh", "id_rsa_devstar.pub")
|
||
privateKeyPath := filepath.Join(homeDir, ".ssh", "id_rsa_devstar")
|
||
if !fileExists(publicKeyPath) || !fileExists(privateKeyPath) {
|
||
err, key := api_service.GenerateNewRSASSHSessionKeyPair()
|
||
if err != nil {
|
||
log.Info("无法创建密钥:", err)
|
||
return ""
|
||
}
|
||
// 确保~/.ssh目录存在
|
||
sshDir := filepath.Join(homeDir, ".ssh")
|
||
if err := os.MkdirAll(sshDir, 0700); err != nil {
|
||
log.Info("无法创建~/.ssh目录:", err)
|
||
return ""
|
||
}
|
||
// 创建密钥文件
|
||
if err := ioutil.WriteFile(privateKeyPath, []byte(key.PrivateKeyPEM), 0600); err != nil {
|
||
log.Info("无法写入私钥文件:", err)
|
||
return ""
|
||
}
|
||
if err := ioutil.WriteFile(publicKeyPath, []byte(key.PublicKeySsh), 0600); err != nil {
|
||
log.Info("无法写入公钥文件:", err)
|
||
return ""
|
||
}
|
||
}
|
||
// 读取公钥文件内容
|
||
publicKeyBytes, err := ioutil.ReadFile(publicKeyPath)
|
||
if err != nil {
|
||
log.Info("Failed to read public key file: %s", err)
|
||
return ""
|
||
}
|
||
|
||
// 将文件内容转换为字符串
|
||
return string(publicKeyBytes)
|
||
}
|
||
func fileExists(filename string) bool {
|
||
info, err := os.Stat(filename)
|
||
if os.IsNotExist(err) {
|
||
return false
|
||
}
|
||
return !info.IsDir()
|
||
}
|
||
|
||
// DeleteRepoDevcontainer 按照 仓库 和/或 用户信息删除 DevContainer(s)
|
||
func DeleteRepoDevcontainer(ctx context.Context, opts *DevcontainersVO.RepoDevcontainerOptions) error {
|
||
if ctx == nil || opts == nil || (opts.Actor == nil && opts.Repository == nil) {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: "construct query parameters",
|
||
Message: "Invalid parameters",
|
||
}
|
||
}
|
||
|
||
// 1. 构造查询条件
|
||
sqlDevcontainerCondition := builder.NewCond()
|
||
if opts.Actor != nil {
|
||
sqlDevcontainerCondition = sqlDevcontainerCondition.And(builder.Eq{"user_id": opts.Actor.ID})
|
||
}
|
||
if opts.Repository != nil {
|
||
sqlDevcontainerCondition = sqlDevcontainerCondition.And(builder.Eq{"repo_id": opts.Repository.ID})
|
||
}
|
||
var devcontainersList []devcontainer_models.DevstarDevcontainer
|
||
|
||
// 2. 开启事务：先获取 devcontainer列表，后删除
|
||
dbTransactionErr := db.WithTx(ctx, func(ctx context.Context) error {
|
||
var err error
|
||
// 2.1 条件查询： user_id 和/或 repo_id
|
||
err = db.GetEngine(ctx).
|
||
Table("devstar_devcontainer").
|
||
Where(sqlDevcontainerCondition).
|
||
Find(&devcontainersList)
|
||
if err != nil {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("find devcontainer(s) with condition '%v'", sqlDevcontainerCondition),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
|
||
// 2.2 空列表，直接结束事务（由于前一个操作只是查询，所以回滚事务不会导致数据不一致问题）
|
||
if len(devcontainersList) == 0 {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("find devcontainer(s) with condition '%v'", sqlDevcontainerCondition),
|
||
Message: "No DevContainer found",
|
||
}
|
||
}
|
||
|
||
// 2.3 条件删除： user_id 和/或 repo_id
|
||
_, err = db.GetEngine(ctx).
|
||
Table("devstar_devcontainer").
|
||
Where(sqlDevcontainerCondition).
|
||
Delete()
|
||
if err != nil {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: fmt.Sprintf("MARK devcontainer(s) as DELETED with condition '%v'", sqlDevcontainerCondition),
|
||
Message: err.Error(),
|
||
}
|
||
}
|
||
|
||
return nil
|
||
})
|
||
|
||
if dbTransactionErr != nil {
|
||
return dbTransactionErr
|
||
}
|
||
|
||
// 3. 后台启动一个goroutine慢慢回收 Dev Container 资源 (如果回收失败，将会产生孤儿 Dev Container，只能管理员手动识别、删除)
|
||
go func() {
|
||
// 注意：由于执行删除 k8s 资源 与 数据库交互和Web页面更新是异步的，因此在 goroutine 中必须重新创建 context，否则报错：
|
||
// Delete "https://192.168.49.2:8443/apis/devcontainer.devstar.cn/v1/...": context canceled
|
||
isolatedContextToPurgeK8sResource, cancelFunc := context.WithCancel(context.Background())
|
||
defer cancelFunc()
|
||
|
||
_ = purgeDevcontainersResource(&isolatedContextToPurgeK8sResource, &devcontainersList)
|
||
}()
|
||
|
||
return dbTransactionErr
|
||
}
|
||
|
||
// getSanitizedDevcontainerName 辅助获取当前用户在当前仓库创建的 devContainer名称
|
||
// DevContainer命名规则： 用户名、仓库名各由小于15位的小写字母和数字组成，中间使用'-'分隔，后面使用'-'分隔符拼接32位UUID小写字母数字字符串
|
||
func getSanitizedDevcontainerName(username, repoName string) string {
|
||
regexpNonAlphaNum := regexp.MustCompile(`[^a-zA-Z0-9]`)
|
||
sanitizedUsername := regexpNonAlphaNum.ReplaceAllString(username, "")
|
||
sanitizedRepoName := regexpNonAlphaNum.ReplaceAllString(repoName, "")
|
||
if len(sanitizedUsername) > 15 {
|
||
sanitizedUsername = strings.ToLower(sanitizedUsername[:15])
|
||
}
|
||
if len(sanitizedRepoName) > 31 {
|
||
sanitizedRepoName = strings.ToLower(sanitizedRepoName[:31])
|
||
}
|
||
newUUID, _ := uuid.NewUUID()
|
||
uuidStr := newUUID.String()
|
||
uuidStr = regexpNonAlphaNum.ReplaceAllString(uuidStr, "")[:15]
|
||
return fmt.Sprintf("%s-%s-%s", sanitizedUsername, sanitizedRepoName, uuidStr)
|
||
}
|
||
|
||
// purgeDevcontainersResource 辅助函数，用于goroutine后台执行，回收DevContainer资源
|
||
func purgeDevcontainersResource(ctx *context.Context, devcontainersList *[]devcontainer_models.DevstarDevcontainer) error {
|
||
// 1. 检查 DevContainer 功能是否启用，若禁用，则直接结束，不会真正执行删除操作
|
||
if !setting.Devcontainer.Enabled {
|
||
// 如果用户设置禁用 DevContainer，无法删除资源，会直接忽略，而数据库相关记录会继续清空、不会发生回滚
|
||
log.Warn("Orphan DevContainers in namespace `%s` left undeleted: %v", setting.Devcontainer.Namespace, devcontainersList)
|
||
return nil
|
||
}
|
||
// 2. 根据配置文件中指定的 DevContainer Agent 派遣创建任务
|
||
switch setting.Devcontainer.Agent {
|
||
case setting.KUBERNETES:
|
||
return devcontainer_k8s_agent_service.AssignDevcontainerDeletion2K8sOperator(ctx, devcontainersList)
|
||
case setting.DOCKER:
|
||
return docker.AssignDevcontainerDeletionDockerOperator(ctx, devcontainersList)
|
||
default:
|
||
// 未知 Agent，直接报错
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: "dispatch DevContainer deletion",
|
||
Message: fmt.Sprintf("unknown DevContainer agent `%s`, please check your config files", setting.Devcontainer.Agent),
|
||
}
|
||
}
|
||
}
|
||
|
||
// claimDevcontainerResource 分发创建 DevContainer 任务到配置文件指定的执行器
|
||
func claimDevcontainerResource(ctx *context.Context, newDevContainer *devcontainer_service_dto.CreateDevcontainerDTO) error {
|
||
// 1. 检查 DevContainer 功能是否启用，若禁用，则直接结束
|
||
if !setting.Devcontainer.Enabled {
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: "Check for DevContainer functionality switch",
|
||
Message: "DevContainer is disabled globally, please check your configuration files",
|
||
}
|
||
}
|
||
|
||
// 2. 根据配置文件中指定的 DevContainer Agent 派遣创建任务
|
||
switch setting.Devcontainer.Agent {
|
||
case setting.KUBERNETES, "k8s":
|
||
// k8s Operator
|
||
return devcontainer_k8s_agent_service.AssignDevcontainerCreation2K8sOperator(ctx, newDevContainer)
|
||
case setting.DOCKER:
|
||
return docker.AssignDevcontainerCreationDockerOperator(ctx, newDevContainer)
|
||
default:
|
||
// 未知 Agent，直接报错
|
||
return devcontainer_models_errors.ErrFailedToOperateDevstarDevcontainerDB{
|
||
Action: "dispatch DevContainer creation",
|
||
Message: fmt.Sprintf("unknown DevContainer agent `%s`, please check your config files", setting.Devcontainer.Agent),
|
||
}
|
||
}
|
||
}
|
||
|
||
// GetDefaultDevcontainerImageFromRepoDevcontainerJSON 从 .devcontainer/devcontainer.json 中获取 devContainer image
|
||
func GetDefaultDevcontainerImageFromRepoDevcontainerJSON(ctx context.Context, repo *repo_model.Repository) string {
|
||
|
||
// 1. 获取默认分支名
|
||
branchName := repo.DefaultBranch
|
||
if len(branchName) == 0 {
|
||
branchName = setting.Devcontainer.DefaultGitBranchName
|
||
}
|
||
|
||
// 2. 打开默认分支
|
||
gitRepoEntity, err := git_module.OpenRepository(ctx, repo.RepoPath())
|
||
if err != nil {
|
||
log.Error("Failed to open repository %s: %v", repo.RepoPath(), err)
|
||
return setting.Devcontainer.DefaultDevcontainerImageName
|
||
}
|
||
defer func(gitRepoEntity *git_module.Repository) {
|
||
_ = gitRepoEntity.Close()
|
||
}(gitRepoEntity)
|
||
|
||
// 3. 获取分支名称
|
||
commit, err := gitRepoEntity.GetBranchCommit(branchName)
|
||
if err != nil {
|
||
return setting.Devcontainer.DefaultDevcontainerImageName
|
||
}
|
||
|
||
// 4. 读取 .devcontainer/devcontainer.json 文件
|
||
const maxDevcontainerJSONSize = 10 * 1024 * 1024 // 设置最大允许的文件大小 10MB
|
||
devcontainerJSONContent, err := commit.GetFileContent(".devcontainer/devcontainer.json", maxDevcontainerJSONSize)
|
||
if err != nil {
|
||
log.Error("Failed to get .devcontainer/devcontainer.json file: %v", err)
|
||
return setting.Devcontainer.DefaultDevcontainerImageName
|
||
}
|
||
|
||
// 5. 解析 JSON
|
||
devContainerJSON, err := devcontainer_models.Unmarshal(devcontainerJSONContent)
|
||
if err != nil {
|
||
log.Error("Failed to unmarshal .devcontainer/devcontainer.json: %v", err)
|
||
return setting.Devcontainer.DefaultDevcontainerImageName
|
||
}
|
||
|
||
// 6. 解析并返回
|
||
if len(devContainerJSON.Image) == 0 {
|
||
return setting.Devcontainer.DefaultDevcontainerImageName
|
||
}
|
||
return devContainerJSON.Image
|
||
}
|