Skip to content

Commit

Permalink
feat: Uniform compression method (#7581)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ssongliu
ssongliu authored Dec 27, 2024
1 parent 554d2ab commit 4d548ad
Show file tree
Hide file tree
Showing 14 changed files with 83 additions and 173 deletions.
8 changes: 4 additions & 4 deletions agent/app/service/backup_app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
}

appPath := install.GetPath()
if err := handleTar(appPath, tmpDir, "app.tar.gz", excludes, ""); err != nil {
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "app.tar.gz"), "", excludes); err != nil {
return err
}

Expand All @@ -170,7 +170,7 @@ func handleAppBackup(install *model.AppInstall, parentTask *task.Task, backupDir
}
}
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
Expand Down Expand Up @@ -200,7 +200,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF

recoverApp := func(t *task.Task) error {
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand Down Expand Up @@ -307,7 +307,7 @@ func handleAppRecover(install *model.AppInstall, parentTask *task.Task, recoverF

deCompressName := i18n.GetWithName("DeCompressFile", "app.tar.gz")
t.LogStart(deCompressName)
if err := handleUnTar(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
if err := fileOp.TarGzExtractPro(tmpPath+"/app.tar.gz", install.GetAppPath(), ""); err != nil {
t.LogFailedWithErr(deCompressName, err)
_ = fileOp.DeleteDir(appDir)
_ = fileOp.Rename(backPath, appDir)
Expand Down
7 changes: 4 additions & 3 deletions agent/app/service/backup_mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,13 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
if strings.HasSuffix(fileName, ".tar.gz") {
fileNameItem := time.Now().Format(constant.DateTimeSlimLayout)
dstDir := fmt.Sprintf("%s/%s", path.Dir(req.File), fileNameItem)
if _, err := os.Stat(dstDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
fileOp := files.NewFileOp()
if !fileOp.Stat(dstDir) {
if err := fileOp.CreateDir(dstDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir, ""); err != nil {
if err := fileOp.TarGzExtractPro(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
7 changes: 4 additions & 3 deletions agent/app/service/backup_postgresql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,13 @@ func (u *BackupService) PostgresqlRecoverByUpload(req dto.CommonRecover) error {
if strings.HasSuffix(fileName, ".tar.gz") {
fileNameItem := time.Now().Format(constant.DateTimeSlimLayout)
dstDir := fmt.Sprintf("%s/%s", path.Dir(req.File), fileNameItem)
if _, err := os.Stat(dstDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
fileOp := files.NewFileOp()
if !fileOp.Stat(dstDir) {
if err := fileOp.CreateDir(dstDir, os.ModePerm); err != nil {
return fmt.Errorf("mkdir %s failed, err: %v", dstDir, err)
}
}
if err := handleUnTar(req.File, dstDir, ""); err != nil {
if err := fileOp.TarGzExtractPro(req.File, dstDir, ""); err != nil {
_ = os.RemoveAll(dstDir)
return err
}
Expand Down
4 changes: 2 additions & 2 deletions agent/app/service/backup_redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func handleRedisBackup(redisInfo *repo.RootInfo, parentTask *task.Task, backupDi

if strings.HasSuffix(fileName, ".tar.gz") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data/appendonlydir", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleTar(redisDataDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, redisDataDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
return nil
Expand Down Expand Up @@ -201,7 +201,7 @@ func handleRedisRecover(redisInfo *repo.RootInfo, parentTask *task.Task, recover
}
if appendonly == "yes" && strings.HasPrefix(redisInfo.Version, "7.") {
redisDataDir := fmt.Sprintf("%s/%s/%s/data", constant.AppInstallDir, "redis", redisInfo.Name)
if err := handleUnTar(recoverFile, redisDataDir, secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, redisDataDir, secret); err != nil {
return err
}
} else {
Expand Down
8 changes: 4 additions & 4 deletions agent/app/service/backup_runtime.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, exc
}

appPath := runtime.GetPath()
if err := handleTar(appPath, tmpDir, "runtime.tar.gz", excludes, secret); err != nil {
if err := fileOp.TarGzCompressPro(true, appPath, path.Join(tmpDir, "runtime.tar.gz"), secret, excludes); err != nil {
return err
}
if err := handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
return nil
Expand All @@ -47,7 +47,7 @@ func handleRuntimeBackup(runtime *model.Runtime, backupDir, fileName string, exc
func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback bool, secret string) error {
isOk := false
fileOp := files.NewFileOp()
if err := handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err := fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
Expand Down Expand Up @@ -100,7 +100,7 @@ func handleRuntimeRecover(runtime *model.Runtime, recoverFile string, isRollback
_ = fileOp.Rename(runtimeDir, backPath)
_ = fileOp.CreateDir(runtimeDir, constant.DirPerm)

if err := handleUnTar(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type), secret); err != nil {
if err := fileOp.TarGzExtractPro(tmpPath+"/runtime.tar.gz", fmt.Sprintf("%s/%s", constant.RuntimeDir, runtime.Type), secret); err != nil {
global.LOG.Errorf("handle recover from runtime.tar.gz failed, err: %v", err)
_ = fileOp.DeleteDir(runtimeDir)
_ = fileOp.Rename(backPath, runtimeDir)
Expand Down
8 changes: 4 additions & 4 deletions agent/app/service/backup_website.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
fileOp := files.NewFileOp()
tmpPath := strings.ReplaceAll(recoverFile, ".tar.gz", "")
t.Log(i18n.GetWithName("DeCompressFile", recoverFile))
if err = handleUnTar(recoverFile, path.Dir(recoverFile), secret); err != nil {
if err = fileOp.TarGzExtractPro(recoverFile, path.Dir(recoverFile), secret); err != nil {
return err
}
defer func() {
Expand Down Expand Up @@ -182,7 +182,7 @@ func handleWebsiteRecover(website *model.Website, recoverFile string, isRollback
}
taskName := i18n.GetMsgByKey("TaskRecover") + i18n.GetMsgByKey("websiteDir")
t.Log(taskName)
if err = handleUnTar(fmt.Sprintf("%s/%s.web.tar.gz", tmpPath, website.Alias), GetSitePath(*website, SiteDir), ""); err != nil {
if err = fileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.web.tar.gz", tmpPath, website.Alias), GetSitePath(*website, SiteDir), ""); err != nil {
t.LogFailedWithErr(taskName, err)
return err
}
Expand Down Expand Up @@ -263,10 +263,10 @@ func handleWebsiteBackup(website *model.Website, backupDir, fileName, excludes,

websiteDir := GetSitePath(*website, SiteDir)
t.LogStart(i18n.GetMsgByKey("CompressDir"))
if err = handleTar(websiteDir, tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias), excludes, ""); err != nil {
if err = fileOp.TarGzCompressPro(true, websiteDir, path.Join(tmpDir, fmt.Sprintf("%s.web.tar.gz", website.Alias)), "", excludes); err != nil {
return err
}
if err = handleTar(tmpDir, backupDir, fileName, "", secret); err != nil {
if err = fileOp.TarGzCompressPro(true, tmpDir, path.Join(backupDir, fileName), secret, ""); err != nil {
return err
}
t.Log(i18n.GetWithName("CompressFileSuccess", fileName))
Expand Down
11 changes: 6 additions & 5 deletions agent/app/service/cronjob_backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,7 @@ func loadWebsForJob(cronjob model.Cronjob) []model.Website {
}

func handleBackupLogs(targetDir, fileName string, secret string) error {
fileOp := files.NewFileOp()
websites, err := websiteRepo.List()
if err != nil {
return err
Expand All @@ -338,7 +339,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(logFiles) != 0 {
for i := 0; i < len(logFiles); i++ {
if !logFiles[i].IsDir() {
_ = common.CopyFile(path.Join(itemDir, logFiles[i].Name()), dirItem)
_ = fileOp.CopyFile(path.Join(itemDir, logFiles[i].Name()), dirItem)
}
}
}
Expand All @@ -347,7 +348,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(logFiles2) != 0 {
for i := 0; i < len(logFiles2); i++ {
if !logFiles2[i].IsDir() {
_ = common.CopyFile(path.Join(itemDir2, logFiles2[i].Name()), dirItem)
_ = fileOp.CopyFile(path.Join(itemDir2, logFiles2[i].Name()), dirItem)
}
}
}
Expand All @@ -366,7 +367,7 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(systemLogFiles) != 0 {
for i := 0; i < len(systemLogFiles); i++ {
if !systemLogFiles[i].IsDir() {
_ = common.CopyFile(path.Join(systemLogDir, systemLogFiles[i].Name()), systemDir)
_ = fileOp.CopyFile(path.Join(systemLogDir, systemLogFiles[i].Name()), systemDir)
}
}
}
Expand All @@ -382,13 +383,13 @@ func handleBackupLogs(targetDir, fileName string, secret string) error {
if len(loginLogFiles) != 0 {
for i := 0; i < len(loginLogFiles); i++ {
if !loginLogFiles[i].IsDir() && (strings.HasPrefix(loginLogFiles[i].Name(), "secure") || strings.HasPrefix(loginLogFiles[i].Name(), "auth.log")) {
_ = common.CopyFile(path.Join("/var/log", loginLogFiles[i].Name()), loginDir)
_ = fileOp.CopyFile(path.Join("/var/log", loginLogFiles[i].Name()), loginDir)
}
}
}
global.LOG.Debug("backup ssh log successful!")

if err := handleTar(targetDir, path.Dir(targetDir), fileName, "", secret); err != nil {
if err := fileOp.TarGzCompressPro(true, targetDir, path.Join(path.Dir(targetDir), fileName), secret, ""); err != nil {
return err
}
defer func() {
Expand Down
76 changes: 0 additions & 76 deletions agent/app/service/cronjob_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/ntp"
"github.com/pkg/errors"
)

func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
Expand Down Expand Up @@ -140,81 +139,6 @@ func (u *CronjobService) handleNtpSync() error {
return nil
}

func handleTar(sourceDir, targetDir, name, exclusionRules string, secret string) error {
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
return err
}
}

excludes := strings.Split(exclusionRules, ",")
excludeRules := ""
for _, exclude := range excludes {
if len(exclude) == 0 {
continue
}
excludeRules += " --exclude " + exclude
}
path := ""
if strings.Contains(sourceDir, "/") {
itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "")
aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")]
if len(aheadDir) == 0 {
aheadDir = "/"
}
path += fmt.Sprintf("-C %s %s", aheadDir, itemDir)
} else {
path = sourceDir
}

commands := ""

if len(secret) != 0 {
extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out"
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -print) -zcf %s %s %s %s", sourceDir, " -"+excludeRules, path, extraCmd, targetDir+"/"+name)
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
} else {
itemPrefix := pathUtils.Base(sourceDir)
if itemPrefix == "/" {
itemPrefix = ""
}
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|%s/|') -zcf %s %s %s", sourceDir, "%P\n", itemPrefix, targetDir+"/"+name, excludeRules, path)
global.LOG.Debug(commands)
}
stdout, err := cmd.ExecWithTimeOut(commands, 24*time.Hour)
if err != nil {
if len(stdout) != 0 {
global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err)
return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err)
}
}
return nil
}

func handleUnTar(sourceFile, targetDir string, secret string) error {
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
return err
}
}
commands := ""
if len(secret) != 0 {
extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceFile + " | "
commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1")
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******"))
} else {
commands = fmt.Sprintf("tar zxvfC %s %s", sourceFile, targetDir)
global.LOG.Debug(commands)
}

stdout, err := cmd.ExecWithTimeOut(commands, 24*time.Hour)
if err != nil {
global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
return errors.New(stdout)
}
return nil
}

func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) {
var (
err error
Expand Down
12 changes: 6 additions & 6 deletions agent/app/service/snapshot_create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,38 +248,38 @@ func snapBaseData(snap snapHelper, targetDir string) error {
snap.Task.Log("---------------------- 2 / 8 ----------------------")
snap.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo"))

err := common.CopyFile("/usr/local/bin/1panel-core", targetDir)
err := snap.FileOp.CopyFile("/usr/local/bin/1panel-core", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-core"), err)
if err != nil {
return err
}

err = common.CopyFile("/usr/local/bin/1panel-agent", targetDir)
err = snap.FileOp.CopyFile("/usr/local/bin/1panel-agent", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel-agent"), err)
if err != nil {
return err
}

err = common.CopyFile("/usr/local/bin/1pctl", targetDir)
err = snap.FileOp.CopyFile("/usr/local/bin/1pctl", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
if err != nil {
return err
}

err = common.CopyFile("/etc/systemd/system/1panel.service", targetDir)
err = snap.FileOp.CopyFile("/etc/systemd/system/1panel.service", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
if err != nil {
return err
}

err = common.CopyFile("/etc/systemd/system/1panel-agent.service", targetDir)
err = snap.FileOp.CopyFile("/etc/systemd/system/1panel-agent.service", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel-agent.service"), err)
if err != nil {
return err
}

if snap.FileOp.Stat("/etc/docker/daemon.json") {
err = common.CopyFile("/etc/docker/daemon.json", targetDir)
err = snap.FileOp.CopyFile("/etc/docker/daemon.json", targetDir)
snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err)
if err != nil {
return err
Expand Down
14 changes: 7 additions & 7 deletions agent/init/viper/viper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/1Panel-dev/1Panel/agent/cmd/server/conf"
"github.com/1Panel-dev/1Panel/agent/configs"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/1Panel-dev/1Panel/agent/utils/xpack"
"github.com/fsnotify/fsnotify"
Expand Down Expand Up @@ -65,10 +64,11 @@ func initDir() {
}
global.CONF.System.BaseDir = nodeInfo.BaseDir

_, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.BaseDir, "1panel/docker/compose/"))
global.CONF.System.DataDir, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.BaseDir, "1panel"))
global.CONF.System.Cache, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "cache"))
global.CONF.System.DbPath, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "db"))
global.CONF.System.LogPath, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "log"))
global.CONF.System.TmpDir, _ = common.CreateDirWhenNotExist(true, path.Join(global.CONF.System.DataDir, "tmp"))
fileOp := files.NewFileOp()
_, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.BaseDir, "1panel/docker/compose/"))
global.CONF.System.DataDir, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.BaseDir, "1panel"))
global.CONF.System.Cache, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "cache"))
global.CONF.System.DbPath, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "db"))
global.CONF.System.LogPath, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "log"))
global.CONF.System.TmpDir, _ = fileOp.CreateDirWithPath(true, path.Join(global.CONF.System.DataDir, "tmp"))
}
Loading

0 comments on commit 4d548ad

Please sign in to comment.