diff --git a/ssh_toolkit/command.go b/ssh_toolkit/command.go index 332ff9067e..ef49947657 100644 --- a/ssh_toolkit/command.go +++ b/ssh_toolkit/command.go @@ -18,13 +18,16 @@ func ExecCommandOverSSH(cmd string, // fetch ssh client sshRecord, err := getSSHClient(host, port, user, privateKey) if err != nil { + if isErrorWhenSSHClientNeedToBeRecreated(err) { + DeleteSSHClient(host) + } return err } // create session session, err := getSSHSessionWithTimeout(sshRecord, sessionTimeoutSeconds) if err != nil { if isErrorWhenSSHClientNeedToBeRecreated(err) { - deleteSSHClient(host) + DeleteSSHClient(host) } return err } @@ -44,7 +47,18 @@ func ExecCommandOverSSH(cmd string, session.Stdout = stdoutBuf session.Stderr = stderrBuf // run command - return session.Run(cmd) + err = session.Run(cmd) + if err != nil { + if isErrorWhenSSHClientNeedToBeRecreated(err) { + DeleteSSHClient(host) + } + if isErrorWhenSSHClientNeedToBeRecreated(errors.New(stderrBuf.String())) { + DeleteSSHClient(host) + return fmt.Errorf("%s - %s", err, stderrBuf.String()) + } + return err + } + return nil } // private functions diff --git a/ssh_toolkit/errors.go b/ssh_toolkit/errors.go index 6f7a594075..926be98b03 100644 --- a/ssh_toolkit/errors.go +++ b/ssh_toolkit/errors.go @@ -9,11 +9,14 @@ var errorsWhenSSHClientNeedToBeRecreated = []string{ "failed to dial ssh", "handshake failed", "unable to authenticate", + "connection refused", + "use of closed network connection", "rejected: too many authentication failures", "rejected: connection closed by remote host", "rejected: connect failed", "open failed", "handshake failed", + "EOF", } func isErrorWhenSSHClientNeedToBeRecreated(err error) bool { diff --git a/ssh_toolkit/net_conn.go b/ssh_toolkit/net_conn.go index c92d28db87..eddc153ef6 100644 --- a/ssh_toolkit/net_conn.go +++ b/ssh_toolkit/net_conn.go @@ -14,12 +14,15 @@ func NetConnOverSSH( // fetch ssh client sshRecord, err := getSSHClient(host, port, user, privateKey) if err != nil { + if isErrorWhenSSHClientNeedToBeRecreated(err) { + DeleteSSHClient(host) + } return nil, err } // create net connection conn, err := dialWithTimeout(sshRecord, network, address, time.Duration(netTimeoutSeconds)*time.Second) if err != nil && isErrorWhenSSHClientNeedToBeRecreated(err) { - deleteSSHClient(host) + DeleteSSHClient(host) } return conn, err } diff --git a/ssh_toolkit/pool.go b/ssh_toolkit/pool.go index 661b103d42..bb876f1aec 100644 --- a/ssh_toolkit/pool.go +++ b/ssh_toolkit/pool.go @@ -68,7 +68,7 @@ func newSSHClient(host string, port int, user string, privateKey string, timeout signer, err := ssh.ParsePrivateKey([]byte(privateKey)) if err != nil { sshClientRecord.mutex.Unlock() - deleteSSHClient(host) + DeleteSSHClient(host) return nil, err } config := &ssh.ClientConfig{ @@ -82,7 +82,7 @@ func newSSHClient(host string, port int, user string, privateKey string, timeout client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", host, port), config) if err != nil { sshClientRecord.mutex.Unlock() - deleteSSHClient(host) + DeleteSSHClient(host) return nil, err } sshClientRecord.client = client @@ -90,7 +90,7 @@ func newSSHClient(host string, port int, user string, privateKey string, timeout return client, nil } -func deleteSSHClient(host string) { +func DeleteSSHClient(host string) { sshClientPool.mutex.Lock() clientEntry, ok := sshClientPool.clients[host] if ok { diff --git a/swiftwave_service/core/types.go b/swiftwave_service/core/types.go index e50f024efe..396bb7aeaf 100644 --- a/swiftwave_service/core/types.go +++ b/swiftwave_service/core/types.go @@ -214,6 +214,7 @@ var RequiredServerDependencies = []string{ "git", "tar", "nfs", + "cifs", "docker", } @@ -224,6 +225,7 @@ var DependencyCheckCommands = map[string]string{ "git": "which git", "tar": "which tar", "nfs": "which nfsstat", + "cifs": "which mount.cifs", "docker": "which docker", } @@ -234,6 +236,7 @@ var DebianDependenciesInstallCommands = map[string]string{ "git": "apt install -y git", "tar": "apt install -y tar", "nfs": "apt install -y nfs-common", + "cifs": "apt install -y cifs-utils", "docker": "curl -fsSL get.docker.com | sh -", } var FedoraDependenciesInstallCommands = map[string]string{ @@ -243,6 +246,7 @@ var FedoraDependenciesInstallCommands = map[string]string{ "git": "dnf install -y git", "tar": "dnf install -y tar", "nfs": "dnf install -y nfs-utils", + "cifs": "dnf install -y cifs-utils", "docker": "curl -fsSL get.docker.com | sh -", } diff --git a/swiftwave_service/cronjob/server_status_monitor.go b/swiftwave_service/cronjob/server_status_monitor.go index 75b1064e80..e3ba268ef2 100644 --- a/swiftwave_service/cronjob/server_status_monitor.go +++ b/swiftwave_service/cronjob/server_status_monitor.go @@ -35,6 +35,9 @@ func (m Manager) monitorServerStatus() { continue } go func(server core.Server) { + if server.Status == core.ServerOffline { + ssh_toolkit.DeleteSSHClient(server.HostName) + } if m.isServerOnline(server) { err = core.MarkServerAsOnline(&m.ServiceManager.DbClient, &server) if err != nil { @@ -60,7 +63,8 @@ func (m Manager) isServerOnline(server core.Server) bool { for i := 0; i < 3; i++ { cmd := "echo ok" stdoutBuf := new(bytes.Buffer) - err := ssh_toolkit.ExecCommandOverSSH(cmd, stdoutBuf, nil, 3, server.IP, server.SSHPort, server.User, m.Config.SystemConfig.SshPrivateKey) + stderrBuf := new(bytes.Buffer) + err := ssh_toolkit.ExecCommandOverSSH(cmd, stdoutBuf, stderrBuf, 3, server.IP, server.SSHPort, server.User, m.Config.SystemConfig.SshPrivateKey) if err != nil { continue } diff --git a/swiftwave_service/db/custom_logger.go b/swiftwave_service/db/custom_logger.go new file mode 100644 index 0000000000..f6c948f516 --- /dev/null +++ b/swiftwave_service/db/custom_logger.go @@ -0,0 +1,124 @@ +package db + +import ( + "context" + "errors" + "fmt" + "github.com/swiftwave-org/swiftwave/swiftwave_service/logger" + gormlogger "gorm.io/gorm/logger" + "path/filepath" + "runtime" + "strconv" + "time" +) + +// newCustomLogger initialize logger +func newCustomLogger(writer gormlogger.Writer, config gormlogger.Config) gormlogger.Interface { + var ( + infoStr = "%s: [info] " + warnStr = "%s: [warn] " + errStr = "%s: [error] " + traceStr = "%s: [trace] [%.3fms] [rows:%v] %s" + traceWarnStr = "%s: [trace] %s\n[%.3fms] [rows:%v] %s" + traceErrStr = "%s: [trace] %s\n[%.3fms] [rows:%v] %s" + ) + + if config.Colorful { + logger.DatabaseLogger.Printf("ignoring colorful logger") + } + + return &customLogger{ + Writer: writer, + Config: config, + infoStr: infoStr, + warnStr: warnStr, + errStr: errStr, + traceStr: traceStr, + traceWarnStr: traceWarnStr, + traceErrStr: traceErrStr, + } +} + +type customLogger struct { + gormlogger.Writer + gormlogger.Config + infoStr, warnStr, errStr string + traceStr, traceErrStr, traceWarnStr string +} + +// LogMode log mode +func (l *customLogger) LogMode(level gormlogger.LogLevel) gormlogger.Interface { + customLogger := *l + customLogger.LogLevel = level + return &customLogger +} + +// Info print info +func (l *customLogger) Info(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gormlogger.Info { + l.Printf(l.infoStr+msg, append([]interface{}{fetchCaller()}, data...)...) + } +} + +// Warn print warn messages +func (l *customLogger) Warn(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gormlogger.Warn { + l.Printf(l.warnStr+msg, append([]interface{}{fetchCaller()}, data...)...) + } +} + +// Error print error messages +func (l *customLogger) Error(ctx context.Context, msg string, data ...interface{}) { + if l.LogLevel >= gormlogger.Error { + l.Printf(l.errStr+msg, append([]interface{}{fetchCaller()}, data...)...) + } +} + +// Trace print sql message +func (l *customLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + if l.LogLevel <= gormlogger.Silent { + return + } + + elapsed := time.Since(begin) + switch { + case err != nil && l.LogLevel >= gormlogger.Error && (!errors.Is(err, gormlogger.ErrRecordNotFound) || !l.IgnoreRecordNotFoundError): + sql, rows := fc() + if rows == -1 { + l.Printf(l.traceErrStr, fetchCaller(), err, float64(elapsed.Nanoseconds())/1e6, "-", sql) + } else { + l.Printf(l.traceErrStr, fetchCaller(), err, float64(elapsed.Nanoseconds())/1e6, rows, sql) + } + case elapsed > l.SlowThreshold && l.SlowThreshold != 0 && l.LogLevel >= gormlogger.Warn: + sql, rows := fc() + slowLog := fmt.Sprintf("SLOW SQL >= %v", l.SlowThreshold) + if rows == -1 { + l.Printf(l.traceWarnStr, fetchCaller(), slowLog, float64(elapsed.Nanoseconds())/1e6, "-", sql) + } else { + l.Printf(l.traceWarnStr, fetchCaller(), slowLog, float64(elapsed.Nanoseconds())/1e6, rows, sql) + } + case l.LogLevel == gormlogger.Info: + sql, rows := fc() + if rows == -1 { + l.Printf(l.traceStr, fetchCaller(), float64(elapsed.Nanoseconds())/1e6, "-", sql) + } else { + l.Printf(l.traceStr, fetchCaller(), float64(elapsed.Nanoseconds())/1e6, rows, sql) + } + } +} + +// ParamsFilter filter params +func (l *customLogger) ParamsFilter(ctx context.Context, sql string, params ...interface{}) (string, []interface{}) { + if l.Config.ParameterizedQueries { + return sql, nil + } + return sql, params +} + +func fetchCaller() string { + _, file, line, ok := runtime.Caller(4) + if !ok { + return "" + } + return filepath.Base(file) + ":" + strconv.Itoa(line) +} diff --git a/swiftwave_service/db/migrations/20240413191732_init.up.sql b/swiftwave_service/db/migrations/20240413191732_init.up.sql index b17fc8e86f..767ab5dcf8 100644 --- a/swiftwave_service/db/migrations/20240413191732_init.up.sql +++ b/swiftwave_service/db/migrations/20240413191732_init.up.sql @@ -76,7 +76,6 @@ CREATE TABLE "public"."servers" ( "ip" text NULL, "host_name" text NULL, "user" text NULL, - "ssh_port" bigint NULL DEFAULT 22, "schedule_deployments" boolean NULL DEFAULT true, "docker_unix_socket_path" text NULL, "swarm_mode" text NULL, diff --git a/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.down.sql b/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.down.sql new file mode 100644 index 0000000000..f872fe9bed --- /dev/null +++ b/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.down.sql @@ -0,0 +1,2 @@ +-- reverse: modify "servers" table +ALTER TABLE "public"."servers" DROP COLUMN "ssh_port"; diff --git a/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.up.sql b/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.up.sql new file mode 100644 index 0000000000..4ce57e2b23 --- /dev/null +++ b/swiftwave_service/db/migrations/20240414051823_server_custom_ssh_port_added.up.sql @@ -0,0 +1,2 @@ +-- modify "servers" table +ALTER TABLE "public"."servers" ADD COLUMN "ssh_port" bigint NULL DEFAULT 22; diff --git a/swiftwave_service/db/migrations/atlas.sum b/swiftwave_service/db/migrations/atlas.sum index 648e66ce19..319298d200 100644 --- a/swiftwave_service/db/migrations/atlas.sum +++ b/swiftwave_service/db/migrations/atlas.sum @@ -1,5 +1,7 @@ -h1:6dZWAUCl1AcaOu5dZJdHaNZ3zngD6MC28BhA19NtHsU= +h1:icdTT4yZ8uFtUOwQFRGT+K1FgTqhqKnIjnOOrTRksdo= 20240413191732_init.down.sql h1:HoitObGwuKF/akF4qg3dol2FfNTLCEuf6wHYDuCez8I= -20240413191732_init.up.sql h1:SSfeL4XTzfkPhdTfo6ElhmZCvlDvESyB4ZLqJbvMkM0= -20240415051843_cifs_config_added.down.sql h1:bpHfK0AkgKiBDjOie+9t/u7uxvjnkSN9skIA85ydtSM= -20240415051843_cifs_config_added.up.sql h1:zgeZR4RgcR+NLCaA1M377m5wMUXNC+gaIsYu9dOLhMM= +20240413191732_init.up.sql h1:USKdQx/yTz1KJ0+mDwYGhKm3WzX7k+I9+6B6SxImwaE= +20240414051823_server_custom_ssh_port_added.down.sql h1:IC1DFQBQceTPTRdZOo5/WqytH+ZbgcKrQuMCkhArF/0= +20240414051823_server_custom_ssh_port_added.up.sql h1:e8WCM91vs1tYOSNujP3H8VL5sdIgL3zJCUO2tJJ4V4A= +20240415051843_cifs_config_added.down.sql h1:mpfh0mD7rG8QzLr+O2lURFSsmVjkZabQZgDqxa4A00M= +20240415051843_cifs_config_added.up.sql h1:O5MSdlTbkma6mtjfsiOKgvuMMnndr2azAq/QnKVbPv8= diff --git a/swiftwave_service/db/utils.go b/swiftwave_service/db/utils.go index bf8b0fb1ac..1cb89fd2b5 100644 --- a/swiftwave_service/db/utils.go +++ b/swiftwave_service/db/utils.go @@ -25,12 +25,23 @@ func GetClient(config *local_config.Config, maxRetry uint) (*gorm.DB, error) { if config.IsDevelopmentMode { logLevel = gormlogger.Info } + ignoreRecordNotFoundError := true + if config.IsDevelopmentMode { + ignoreRecordNotFoundError = false + } + parameterizedQueries := true + if config.IsDevelopmentMode { + parameterizedQueries = false + } + gormConfig := &gorm.Config{ SkipDefaultTransaction: true, - Logger: gormlogger.New(logger.DatabaseLogger, gormlogger.Config{ - SlowThreshold: 500 * time.Millisecond, - Colorful: false, - LogLevel: logLevel, + Logger: newCustomLogger(logger.DatabaseLogger, gormlogger.Config{ + SlowThreshold: 500 * time.Millisecond, + Colorful: false, + LogLevel: logLevel, + IgnoreRecordNotFoundError: ignoreRecordNotFoundError, + ParameterizedQueries: parameterizedQueries, }), } currentRetry := uint(0) diff --git a/swiftwave_service/graphql/graphql_object_mapper.go b/swiftwave_service/graphql/graphql_object_mapper.go index edcdbafb9b..55f6a7f9cc 100644 --- a/swiftwave_service/graphql/graphql_object_mapper.go +++ b/swiftwave_service/graphql/graphql_object_mapper.go @@ -80,22 +80,30 @@ func persistentVolumeToGraphqlObject(record *core.PersistentVolume) *model.Persi // persistentVolumeInputToDatabaseObject : converts PersistentVolumeInput to PersistentVolumeDatabaseObject func persistentVolumeInputToDatabaseObject(record *model.PersistentVolumeInput) *core.PersistentVolume { - return &core.PersistentVolume{ - Name: record.Name, - Type: core.PersistentVolumeType(record.Type), - NFSConfig: core.NFSConfig{ + nfsConfig := core.NFSConfig{} + if record.Type == model.PersistentVolumeTypeNfs { + nfsConfig = core.NFSConfig{ Host: record.NfsConfig.Host, Path: record.NfsConfig.Path, Version: record.NfsConfig.Version, - }, - CIFSConfig: core.CIFSConfig{ + } + } + cifsConfig := core.CIFSConfig{} + if record.Type == model.PersistentVolumeTypeCifs { + cifsConfig = core.CIFSConfig{ Share: record.CifsConfig.Share, Host: record.CifsConfig.Host, Username: record.CifsConfig.Username, Password: record.CifsConfig.Password, FileMode: record.CifsConfig.FileMode, DirMode: record.CifsConfig.DirMode, - }, + } + } + return &core.PersistentVolume{ + Name: record.Name, + Type: core.PersistentVolumeType(record.Type), + NFSConfig: nfsConfig, + CIFSConfig: cifsConfig, } } diff --git a/swiftwave_service/logger/types.go b/swiftwave_service/logger/types.go index 7ad16c55f6..da50481d7c 100644 --- a/swiftwave_service/logger/types.go +++ b/swiftwave_service/logger/types.go @@ -7,7 +7,7 @@ import ( "strconv" ) -var DatabaseLogger = log.New(os.Stdout, "[DATABASE] ", log.Ldate|log.Ltime|log.LUTC|log.Lshortfile) +var DatabaseLogger = log.New(os.Stdout, "[DATABASE] ", log.Ldate|log.Ltime|log.LUTC) var DatabaseLoggerError = log.New(os.Stdout, "[DATABASE] ", log.Ldate|log.Ltime|log.LUTC|log.Lshortfile) var CronJobLogger = log.New(os.Stdout, "[CRONJOB] ", log.Ldate|log.Ltime|log.LUTC|log.Lshortfile) var CronJobLoggerError = log.New(os.Stdout, "[CRONJOB] ", log.Ldate|log.Ltime|log.LUTC|log.Lshortfile)