Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: The result of the scheduled task execution is based on the task #7586

Merged
merged 1 commit into from
Dec 28, 2024
Merged

feat: The result of the scheduled task execution is based on the task #7586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent/app/api/v2/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) {
return
}

if err := snapshotService.SnapshotCreate(req); err != nil {
if err := snapshotService.SnapshotCreate(req, false); err != nil {
helper.InternalServer(c, err)
return
}
Expand Down
1 change: 1 addition & 0 deletions agent/app/dto/cronjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ type SearchRecord struct {

type Record struct {
ID uint `json:"id"`
TaskID string `json:"taskID"`
StartTime string `json:"startTime"`
Records string `json:"records"`
Status string `json:"status"`
Expand Down
2 changes: 2 additions & 0 deletions agent/app/repo/cronjob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/google/uuid"
"gorm.io/gorm"
)

Expand Down Expand Up @@ -146,6 +147,7 @@ func (u *CronjobRepo) StartRecords(cronjobID uint, targetPath string) model.JobR
var record model.JobRecords
record.StartTime = time.Now()
record.CronjobID = cronjobID
record.TaskID = uuid.New().String()
record.Status = constant.StatusWaiting
if err := global.DB.Create(&record).Error; err != nil {
global.LOG.Errorf("create record status failed, err: %v", err)
Expand Down
38 changes: 28 additions & 10 deletions agent/app/service/cronjob_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/pkg/errors"
)

func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, taskID string) error {
var apps []model.AppInstall
if cronjob.AppID == "all" {
apps, _ = appInstallRepo.ListBy()
Expand Down Expand Up @@ -46,7 +46,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) e
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name))
record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleAppBackup(&app, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, ""); err != nil {
if err := handleAppBackup(&app, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
Expand All @@ -63,7 +63,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) e
return nil
}

func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time, taskID string) error {
webs := loadWebsForJob(cronjob)
if len(webs) == 0 {
return errors.New("no such website in database!")
Expand All @@ -82,7 +82,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("website/%s", web.PrimaryDomain))
record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", web.PrimaryDomain, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleWebsiteBackup(&web, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, ""); err != nil {
if err := handleWebsiteBackup(&web, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
Expand All @@ -99,7 +99,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return nil
}

func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time, taskID string) error {
dbs := loadDbsForJob(cronjob)
if len(dbs) == 0 {
return errors.New("no such db in database!")
Expand All @@ -120,11 +120,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
return err
}
} else {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
return err
}
}
Expand Down Expand Up @@ -212,11 +212,15 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
return nil
}

func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time, taskID string) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
}
itemData, err := NewISnapshotService().LoadSnapshotData()
if err != nil {
return err
}

var record model.BackupRecord
record.From = "cronjob"
Expand All @@ -227,14 +231,28 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
record.FileDir = "system_snapshot"

versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
scope := "core"
if !global.IsMaster {
scope = "agent"
}
req := dto.SnapshotCreate{
Name: fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Secret: cronjob.Secret,
TaskID: taskID,

SourceAccountIDs: record.SourceAccountIDs,
DownloadAccountID: cronjob.DownloadAccountID,
AppData: itemData.AppData,
PanelData: itemData.PanelData,
BackupData: itemData.BackupData,
WithMonitorData: true,
WithLoginLog: true,
WithOperationLog: true,
WithSystemLog: true,
WithTaskLog: true,
}
if err := NewISnapshotService().HandleSnapshot(req); err != nil {

if err := NewISnapshotService().SnapshotCreate(req, true); err != nil {
return err
}
record.FileName = req.Name + ".tar.gz"
Expand Down
156 changes: 92 additions & 64 deletions agent/app/service/cronjob_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
Expand All @@ -31,36 +32,26 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
if len(cronjob.Script) == 0 {
return
}
record.Records = u.generateLogsPath(*cronjob, record.StartTime)
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleShell(*cronjob, record.Records)
u.removeExpiredLog(*cronjob)
err = u.handleShell(*cronjob, record.TaskID)
case "curl":
if len(cronjob.URL) == 0 {
return
}
record.Records = u.generateLogsPath(*cronjob, record.StartTime)
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = cmd.ExecShell(record.Records, 24*time.Hour, "bash", "-c", "curl", cronjob.URL)
u.removeExpiredLog(*cronjob)
err = u.handleCurl(*cronjob, record.TaskID)
case "ntp":
err = u.handleNtpSync()
u.removeExpiredLog(*cronjob)
err = u.handleNtpSync(*cronjob, record.TaskID)
case "cutWebsiteLog":
var messageItem []string
messageItem, record.File, err = u.handleCutWebsiteLog(cronjob, record.StartTime)
message = []byte(strings.Join(messageItem, "\n"))
case "clean":
messageItem := ""
messageItem, err = u.handleSystemClean()
message = []byte(messageItem)
u.removeExpiredLog(*cronjob)
err = u.handleSystemClean(*cronjob, record.TaskID)
case "website":
err = u.handleWebsite(*cronjob, record.StartTime)
err = u.handleWebsite(*cronjob, record.StartTime, record.TaskID)
case "app":
err = u.handleApp(*cronjob, record.StartTime)
err = u.handleApp(*cronjob, record.StartTime, record.TaskID)
case "database":
err = u.handleDatabase(*cronjob, record.StartTime)
err = u.handleDatabase(*cronjob, record.StartTime, record.TaskID)
case "directory":
if len(cronjob.SourceDir) == 0 {
return
Expand All @@ -70,7 +61,7 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
err = u.handleSystemLog(*cronjob, record.StartTime)
case "snapshot":
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleSnapshot(*cronjob, record.StartTime)
err = u.handleSnapshot(*cronjob, record.StartTime, record.TaskID)
}

if err != nil {
Expand All @@ -90,53 +81,95 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
}()
}

func (u *CronjobService) handleShell(cronjob model.Cronjob, logPath string) error {
if len(cronjob.ContainerName) != 0 {
command := "sh"
if len(cronjob.Command) != 0 {
command = cronjob.Command
}
scriptFile, _ := os.ReadFile(cronjob.Script)
return cmd.ExecShell(logPath, 24*time.Hour, "docker", "exec", cronjob.ContainerName, command, "-c", strings.ReplaceAll(string(scriptFile), "\"", "\\\""))
}
if len(cronjob.Executor) == 0 {
cronjob.Executor = "bash"
func (u *CronjobService) handleShell(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
if cronjob.ScriptMode == "input" {
fileItem := pathUtils.Join(global.CONF.System.BaseDir, "1panel", "task", "shell", cronjob.Name, cronjob.Name+".sh")
_ = os.MkdirAll(pathUtils.Dir(fileItem), os.ModePerm)
shellFile, err := os.OpenFile(fileItem, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, constant.FilePerm)
if err != nil {
return err

taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
if len(cronjob.ContainerName) != 0 {
command := "sh"
if len(cronjob.Command) != 0 {
command = cronjob.Command
}
scriptFile, _ := os.ReadFile(cronjob.Script)
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, "docker", "exec", cronjob.ContainerName, command, "-c", strings.ReplaceAll(string(scriptFile), "\"", "\\\""))
}
defer shellFile.Close()
if _, err := shellFile.WriteString(cronjob.Script); err != nil {
return err
if len(cronjob.Executor) == 0 {
cronjob.Executor = "bash"
}
if cronjob.ScriptMode == "input" {
fileItem := pathUtils.Join(global.CONF.System.BaseDir, "1panel", "task", "shell", cronjob.Name, cronjob.Name+".sh")
_ = os.MkdirAll(pathUtils.Dir(fileItem), os.ModePerm)
shellFile, err := os.OpenFile(fileItem, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, constant.FilePerm)
if err != nil {
return err
}
defer shellFile.Close()
if _, err := shellFile.WriteString(cronjob.Script); err != nil {
return err
}
if len(cronjob.User) == 0 {
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, cronjob.Executor, fileItem)
}
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, fileItem)
}
if len(cronjob.User) == 0 {
return cmd.ExecShell(logPath, 24*time.Hour, cronjob.Executor, fileItem)
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, cronjob.Executor, cronjob.Script)
}
return cmd.ExecShell(logPath, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, fileItem)
}
if len(cronjob.User) == 0 {
return cmd.ExecShell(logPath, 24*time.Hour, cronjob.Executor, cronjob.Script)
}
return cmd.ExecShell(logPath, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, cronjob.Script)
if err := cmd.ExecShellWithTask(taskItem, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, cronjob.Script); err != nil {
return err
}
return nil
},
nil,
)
return taskItem.Execute()
}

func (u *CronjobService) handleNtpSync() error {
ntpServer, err := settingRepo.Get(settingRepo.WithByKey("NtpSite"))
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
ntime, err := ntp.GetRemoteTime(ntpServer.Value)

taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
if err := cmd.ExecShellWithTask(taskItem, 24*time.Hour, "bash", "-c", "curl", cronjob.URL); err != nil {
return err
}
return nil
},
nil,
)
return taskItem.Execute()
}

func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
if err := ntp.UpdateSystemTime(ntime.Format(constant.DateTimeLayout)); err != nil {
return err
}
return nil

taskItem.AddSubTask(i18n.GetMsgByKey("HandleNtpSync"), func(t *task.Task) error {
ntpServer, err := settingRepo.Get(settingRepo.WithByKey("NtpSite"))
if err != nil {
return err
}
taskItem.Logf("ntp server: %s", ntpServer.Value)
ntime, err := ntp.GetRemoteTime(ntpServer.Value)
if err != nil {
return err
}
if err := ntp.UpdateSystemTime(ntime.Format(constant.DateTimeLayout)); err != nil {
return err
}
return nil
}, nil)
return taskItem.Execute()
}

func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) {
Expand Down Expand Up @@ -201,8 +234,13 @@ func backupLogFile(dstFilePath, websiteLogDir string, fileOp files.FileOp) error
return nil
}

func (u *CronjobService) handleSystemClean() (string, error) {
return NewIDeviceService().CleanForCronjob()
func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for system clean failed, err: %v", err)
return err
}
return systemClean(taskItem)
}

func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, accountMap map[string]backupClientHelper, file string) (string, error) {
Expand Down Expand Up @@ -274,16 +312,6 @@ func (u *CronjobService) removeExpiredLog(cronjob model.Cronjob) {
}
}

func (u *CronjobService) generateLogsPath(cronjob model.Cronjob, startTime time.Time) string {
dir := fmt.Sprintf("%s/task/%s/%s", constant.DataDir, cronjob.Type, cronjob.Name)
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
_ = os.MkdirAll(dir, os.ModePerm)
}

path := fmt.Sprintf("%s/%s.log", dir, startTime.Format(constant.DateTimeSlimLayout))
return path
}

func hasBackup(cronjobType string) bool {
return cronjobType == "app" || cronjobType == "database" || cronjobType == "website" || cronjobType == "directory" || cronjobType == "snapshot" || cronjobType == "log"
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The given Go code is designed to be part of an application that manages the execution of various tasks based on different types of cron jobs. The following are some observations made from reviewing the provided code:

  1. There seems to be a small typo where the import statement underlining handleShel has a missing semicolon at the end.

  2. A function removeExpiredLog is called before handling log messages, which suggests it might not be properly integrated into the logic flow.

  3. It would be beneficial to add unit test cases for this class so that we can verify how the functions behave in the context of real-world use cases.

  4. Since some of the lines like if, else if, etc. lack proper formatting or indentation, they could potentially make them more difficult to read and understand especially in larger codebases.

  5. You should consider importing the relevant packages (github.com/go-playground/tabletable) rather than using import (). This will ensure that you're always loading all the necessary versions when you start coding.

To address these discrepancies:

  • Add a space after each comma in the import statements.
  • Correctly indent removeExpiredLog.
  • Importing required library instead of directly stating its name within parentheses is more preferable here as well.

  1. Implement unittests for generateLogsPath() and perhaps also removeExpiredLog().

  2. Ensure consistent naming conventions across functions such as HandleJob, handleNtp_sync, handleShell.

It's important not just to fix syntax errors but to correct potential bugs and improve readability as much as possible. Also, adding tests ensures that changes do not break other parts of your program while improving maintainability and robustness through self-checking tools.

1 change: 0 additions & 1 deletion agent/app/service/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ type IDeviceService interface {

Scan() dto.CleanData
Clean(req []dto.Clean)
CleanForCronjob() (string, error)
}

func NewIDeviceService() IDeviceService {
Expand Down
Loading
Loading