From 43cd32a631158ab1b7109397ad7e7f51271fe09f Mon Sep 17 00:00:00 2001 From: zhengkunwang223 Date: Thu, 26 Dec 2024 18:28:22 +0800 Subject: [PATCH] feat(task): Optimize Container Log Reading --- agent/app/api/v2/container.go | 69 ++---- agent/app/dto/container.go | 9 + agent/app/service/container.go | 144 ++++++------ agent/router/ro_container.go | 3 +- frontend/package.json | 3 +- frontend/src/components/compose-log/index.vue | 163 +------------ .../src/components/container-log/index.vue | 221 ++++++++++++------ frontend/src/components/log-file/index.vue | 1 - frontend/src/components/task-log/index.vue | 3 + frontend/src/main.ts | 4 + .../views/container/container/log/index.vue | 149 +----------- .../views/database/mysql/setting/index.vue | 8 +- .../database/postgresql/setting/index.vue | 8 +- .../src/views/website/website/nginx/index.vue | 14 +- 14 files changed, 289 insertions(+), 510 deletions(-) diff --git a/agent/app/api/v2/container.go b/agent/app/api/v2/container.go index 79cd797661f0..50cd5a69a31a 100644 --- a/agent/app/api/v2/container.go +++ b/agent/app/api/v2/container.go @@ -5,7 +5,6 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/api/v2/helper" "github.com/1Panel-dev/1Panel/agent/app/dto" - "github.com/1Panel-dev/1Panel/agent/global" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) @@ -470,34 +469,6 @@ func (b *BaseApi) Inspect(c *gin.Context) { helper.SuccessWithData(c, result) } -// @Tags Container -// @Summary Container logs -// @Description 容器日志 -// @Param container query string false "容器名称" -// @Param since query string false "时间筛选" -// @Param follow query string false "是否追踪" -// @Param tail query string false "显示行号" -// @Security ApiKeyAuth -// @Router /containers/search/log [post] -func (b *BaseApi) ContainerLogs(c *gin.Context) { - wsConn, err := upGrader.Upgrade(c.Writer, c.Request, nil) - if err != nil { - global.LOG.Errorf("gin context http handler failed, err: %v", err) - return - } - defer wsConn.Close() - - container := c.Query("container") - since := c.Query("since") - follow := c.Query("follow") == "true" - tail := c.Query("tail") - - if err := containerService.ContainerLogs(wsConn, "container", container, since, tail, follow); err != nil { - _ = wsConn.WriteMessage(1, []byte(err.Error())) - return - } -} - // @Description 下载容器日志 // @Router /containers/download/log [post] func (b *BaseApi) DownloadContainerLogs(c *gin.Context) { @@ -707,30 +678,38 @@ func (b *BaseApi) ComposeUpdate(c *gin.Context) { helper.SuccessWithData(c, nil) } -// @Tags Container Compose -// @Summary Container Compose logs -// @Description docker-compose 日志 -// @Param compose query string false "compose 文件地址" +// @Tags Container +// @Summary Container logs +// @Description 容器日志 +// @Param container query string false "容器名称" // @Param since query string false "时间筛选" // @Param follow query string false "是否追踪" // @Param tail query string false "显示行号" // @Security ApiKeyAuth -// @Router /containers/compose/search/log [get] -func (b *BaseApi) ComposeLogs(c *gin.Context) { - wsConn, err := upGrader.Upgrade(c.Writer, c.Request, nil) - if err != nil { - global.LOG.Errorf("gin context http handler failed, err: %v", err) - return - } - defer wsConn.Close() +// @Router /containers/search/log [post] +func (b *BaseApi) ContainerStreamLogs(c *gin.Context) { + c.Header("Content-Type", "text/event-stream") + c.Header("Cache-Control", "no-cache") + c.Header("Connection", "keep-alive") + c.Header("Transfer-Encoding", "chunked") - compose := c.Query("compose") since := c.Query("since") follow := c.Query("follow") == "true" tail := c.Query("tail") - if err := containerService.ContainerLogs(wsConn, "compose", compose, since, tail, follow); err != nil { - _ = wsConn.WriteMessage(1, []byte(err.Error())) - return + container := c.Query("container") + compose := c.Query("compose") + streamLog := dto.StreamLog{ + Compose: compose, + Container: container, + Since: since, + Follow: follow, + Tail: tail, + Type: "container", } + if compose != "" { + streamLog.Type = "compose" + } + + containerService.StreamLogs(c, streamLog) } diff --git a/agent/app/dto/container.go b/agent/app/dto/container.go index 01f0871bf849..590ad6e93e62 100644 --- a/agent/app/dto/container.go +++ b/agent/app/dto/container.go @@ -268,3 +268,12 @@ type ContainerLog struct { Tail uint `json:"tail"` ContainerType string `json:"containerType"` } + +type StreamLog struct { + Compose string + Container string + Since string + Follow bool + Tail string + Type string +} diff --git a/agent/app/service/container.go b/agent/app/service/container.go index a9c04858ceef..0df94e4ad5b8 100644 --- a/agent/app/service/container.go +++ b/agent/app/service/container.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/gin-gonic/gin" "io" "net/http" "net/url" @@ -19,9 +20,6 @@ import ( "sync" "syscall" "time" - "unicode/utf8" - - "github.com/gin-gonic/gin" "github.com/pkg/errors" @@ -44,7 +42,6 @@ import ( "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" - "github.com/gorilla/websocket" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/mem" @@ -74,7 +71,6 @@ type IContainerService interface { ContainerCommit(req dto.ContainerCommit) error ContainerLogClean(req dto.OperationWithName) error ContainerOperation(req dto.ContainerOperation) error - ContainerLogs(wsConn *websocket.Conn, containerType, container, since, tail string, follow bool) error DownloadContainerLogs(containerType, container, since, tail string, c *gin.Context) error ContainerStats(id string) (*dto.ContainerStats, error) Inspect(req dto.InspectReq) (string, error) @@ -87,6 +83,8 @@ type IContainerService interface { Prune(req dto.ContainerPrune) (dto.ContainerPruneReport, error) LoadContainerLogs(req dto.OperationWithNameAndType) string + + StreamLogs(ctx *gin.Context, params dto.StreamLog) } func NewIContainerService() IContainerService { @@ -794,87 +792,87 @@ func (u *ContainerService) ContainerLogClean(req dto.OperationWithName) error { return nil } -func (u *ContainerService) ContainerLogs(wsConn *websocket.Conn, containerType, container, since, tail string, follow bool) error { - defer func() { wsConn.Close() }() - if cmd.CheckIllegal(container, since, tail) { - return buserr.New(constant.ErrCmdIllegal) +func (u *ContainerService) StreamLogs(ctx *gin.Context, params dto.StreamLog) { + messageChan := make(chan string, 1024) + errorChan := make(chan error, 1) + + go collectLogs(params, messageChan, errorChan) + + ctx.Stream(func(w io.Writer) bool { + select { + case msg, ok := <-messageChan: + if !ok { + return false + } + _, err := fmt.Fprintf(w, "data: %v\n\n", msg) + if err != nil { + return false + } + return true + case err := <-errorChan: + _, err = fmt.Fprintf(w, "data: {\"event\": \"error\", \"data\": \"%s\"}\n\n", err.Error()) + if err != nil { + return false + } + return false + case <-ctx.Request.Context().Done(): + return false + } + }) +} + +func collectLogs(params dto.StreamLog, messageChan chan<- string, errorChan chan<- error) { + defer close(messageChan) + defer close(errorChan) + + var cmdArgs []string + if params.Type == "compose" { + cmdArgs = []string{"compose", "-f", params.Compose} } - commandName := "docker" - commandArg := []string{"logs", container} - if containerType == "compose" { - commandArg = []string{"compose", "-f", container, "logs"} + cmdArgs = append(cmdArgs, "logs") + if params.Follow { + cmdArgs = append(cmdArgs, "-f") } - if tail != "0" { - commandArg = append(commandArg, "--tail") - commandArg = append(commandArg, tail) + if params.Tail != "all" { + cmdArgs = append(cmdArgs, "--tail", params.Tail) } - if since != "all" { - commandArg = append(commandArg, "--since") - commandArg = append(commandArg, since) + if params.Since != "all" { + cmdArgs = append(cmdArgs, "--since", params.Since) } - if follow { - commandArg = append(commandArg, "-f") - } - if !follow { - cmd := exec.Command(commandName, commandArg...) - cmd.Stderr = cmd.Stdout - stdout, _ := cmd.CombinedOutput() - if !utf8.Valid(stdout) { - return errors.New("invalid utf8") - } - if err := wsConn.WriteMessage(websocket.TextMessage, stdout); err != nil { - global.LOG.Errorf("send message with log to ws failed, err: %v", err) - } - return nil + if params.Container != "" { + cmdArgs = append(cmdArgs, params.Container) } + cmd := exec.Command("docker", cmdArgs...) - cmd := exec.Command(commandName, commandArg...) stdout, err := cmd.StdoutPipe() if err != nil { - _ = cmd.Process.Signal(syscall.SIGTERM) - return err + errorChan <- fmt.Errorf("failed to get stdout pipe: %v", err) + return } - cmd.Stderr = cmd.Stdout if err := cmd.Start(); err != nil { - _ = cmd.Process.Signal(syscall.SIGTERM) - return err + errorChan <- fmt.Errorf("failed to start command: %v", err) + return } - exitCh := make(chan struct{}) - go func() { - _, wsData, _ := wsConn.ReadMessage() - if string(wsData) == "close conn" { - _ = cmd.Process.Signal(syscall.SIGTERM) - exitCh <- struct{}{} - } - }() - go func() { - buffer := make([]byte, 1024) - for { - select { - case <-exitCh: - return - default: - n, err := stdout.Read(buffer) - if err != nil { - if err == io.EOF { - return - } - global.LOG.Errorf("read bytes from log failed, err: %v", err) - return - } - if !utf8.Valid(buffer[:n]) { - continue - } - if err = wsConn.WriteMessage(websocket.TextMessage, buffer[:n]); err != nil { - global.LOG.Errorf("send message with log to ws failed, err: %v", err) - return - } - } + scanner := bufio.NewScanner(stdout) + lineNumber := 0 + + for scanner.Scan() { + lineNumber++ + message := scanner.Text() + select { + case messageChan <- message: + case <-time.After(time.Second): + errorChan <- fmt.Errorf("message channel blocked") + return } - }() - _ = cmd.Wait() - return nil + } + + if err := scanner.Err(); err != nil { + errorChan <- fmt.Errorf("scanner error: %v", err) + return + } + cmd.Wait() } func (u *ContainerService) DownloadContainerLogs(containerType, container, since, tail string, c *gin.Context) error { diff --git a/agent/router/ro_container.go b/agent/router/ro_container.go index 466c626c65b9..abd4269caa84 100644 --- a/agent/router/ro_container.go +++ b/agent/router/ro_container.go @@ -23,7 +23,7 @@ func (s *ContainerRouter) InitRouter(Router *gin.RouterGroup) { baRouter.POST("/list", baseApi.ListContainer) baRouter.GET("/status", baseApi.LoadContainerStatus) baRouter.GET("/list/stats", baseApi.ContainerListStats) - baRouter.GET("/search/log", baseApi.ContainerLogs) + baRouter.GET("/search/log", baseApi.ContainerStreamLogs) baRouter.POST("/download/log", baseApi.DownloadContainerLogs) baRouter.GET("/limit", baseApi.LoadResourceLimit) baRouter.POST("/clean/log", baseApi.CleanContainerLog) @@ -46,7 +46,6 @@ func (s *ContainerRouter) InitRouter(Router *gin.RouterGroup) { baRouter.POST("/compose/test", baseApi.TestCompose) baRouter.POST("/compose/operate", baseApi.OperatorCompose) baRouter.POST("/compose/update", baseApi.ComposeUpdate) - baRouter.GET("/compose/search/log", baseApi.ComposeLogs) baRouter.GET("/template", baseApi.ListComposeTemplate) baRouter.POST("/template/search", baseApi.SearchComposeTemplate) diff --git a/frontend/package.json b/frontend/package.json index b7e4d0bf2965..3d3b84bfc35d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -55,7 +55,8 @@ "vue-codemirror": "^6.1.1", "vue-demi": "^0.14.6", "vue-i18n": "^9.13.1", - "vue-router": "^4.3.3" + "vue-router": "^4.3.3", + "vue-virtual-scroller": "^2.0.0-beta.8" }, "devDependencies": { "@types/node": "^20.14.8", diff --git a/frontend/src/components/compose-log/index.vue b/frontend/src/components/compose-log/index.vue index 680124357dc4..3b368011f3c8 100644 --- a/frontend/src/components/compose-log/index.vue +++ b/frontend/src/components/compose-log/index.vue @@ -1,7 +1,7 @@ @@ -140,13 +186,34 @@ defineExpose({ .margin-button { margin-left: 20px; } -.selectWidth { - width: 150px; +.fullScreen { + border: none; +} +.tailClass { + width: 20%; + float: left; + margin-left: 20px; +} +.fetchClass { + width: 30%; + float: left; } -.editor-main { - height: calc(100vh - 480px); + +.log-container { + height: calc(100vh - 405px); + overflow-y: auto; + overflow-x: auto; + position: relative; + background-color: #1e1e1e; + margin-top: 10px; +} + +.log-item { + position: absolute; width: 100%; - min-height: 600px; - overflow: auto; + padding: 2px; + color: #f5f5f5; + box-sizing: border-box; + white-space: nowrap; } diff --git a/frontend/src/components/log-file/index.vue b/frontend/src/components/log-file/index.vue index 998453c70b43..ea326e530dfd 100644 --- a/frontend/src/components/log-file/index.vue +++ b/frontend/src/components/log-file/index.vue @@ -218,7 +218,6 @@ const getContent = async (pre: boolean) => { } nextTick(() => { - console.log('pre', pre); if (pre) { logContainer.value.scrollTop = 2000; } else { diff --git a/frontend/src/components/task-log/index.vue b/frontend/src/components/task-log/index.vue index 652ebfebb326..9fe1b118dbd5 100644 --- a/frontend/src/components/task-log/index.vue +++ b/frontend/src/components/task-log/index.vue @@ -43,9 +43,12 @@ const open = ref(false); const showTail = ref(true); const openWithTaskID = (id: string, tail: boolean) => { + console.log('openWithTaskID', id, tail); config.taskID = id; if (!tail) { config.tail = false; + } else { + config.tail = true; } open.value = true; }; diff --git a/frontend/src/main.ts b/frontend/src/main.ts index aff4fdc1792c..ee6670f4a8f1 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -25,6 +25,9 @@ import Fit2CloudPlus from 'fit2cloud-ui-plus'; import * as Icons from '@element-plus/icons-vue'; import hljsVuePlugin from '@highlightjs/vue-plugin'; +import 'vue-virtual-scroller/dist/vue-virtual-scroller.css'; +import VirtualScroller from 'vue-virtual-scroller'; + const app = createApp(App); app.use(hljsVuePlugin); app.component('SvgIcon', SvgIcon); @@ -35,6 +38,7 @@ Object.keys(Icons).forEach((key) => { app.component(key, Icons[key as keyof typeof Icons]); }); +app.use(VirtualScroller); app.use(router); app.use(i18n); app.use(pinia); diff --git a/frontend/src/views/container/container/log/index.vue b/frontend/src/views/container/container/log/index.vue index 1b0e99588a08..43541fea01e7 100644 --- a/frontend/src/views/container/container/log/index.vue +++ b/frontend/src/views/container/container/log/index.vue @@ -12,32 +12,7 @@ diff --git a/frontend/src/views/database/mysql/setting/index.vue b/frontend/src/views/database/mysql/setting/index.vue index 0c66ef3e62a9..8e0c94760309 100644 --- a/frontend/src/views/database/mysql/setting/index.vue +++ b/frontend/src/views/database/mysql/setting/index.vue @@ -86,7 +86,7 @@ - + (), { database: '', }); -const dialogContainerLogRef = ref(); const jumpToConf = async () => { activeName.value = 'conf'; loadMysqlConf(); @@ -275,8 +275,8 @@ const onSaveConf = async () => { return; }; -const loadContainerLog = async (containerID: string) => { - dialogContainerLogRef.value!.acceptParams({ containerID: containerID, container: containerID }); +const loadContainerLog = async (conID: string) => { + containerID.value = conID; }; const loadBaseInfo = async () => { diff --git a/frontend/src/views/database/postgresql/setting/index.vue b/frontend/src/views/database/postgresql/setting/index.vue index d2b6766368c2..c974e62edb2e 100644 --- a/frontend/src/views/database/postgresql/setting/index.vue +++ b/frontend/src/views/database/postgresql/setting/index.vue @@ -46,7 +46,7 @@ - + @@ -109,7 +109,7 @@ const props = withDefaults(defineProps(), { database: '', }); -const dialogContainerLogRef = ref(); +const containerID = ref(''); const jumpToConf = async () => { activeName.value = 'conf'; loadPostgresqlConf(); @@ -181,8 +181,8 @@ const onSaveConf = async () => { return; }; -const loadContainerLog = async (containerID: string) => { - dialogContainerLogRef.value!.acceptParams({ containerID: containerID, container: containerID }); +const loadContainerLog = async (conID: string) => { + containerID.value = conID; }; const loadBaseInfo = async () => { diff --git a/frontend/src/views/website/website/nginx/index.vue b/frontend/src/views/website/website/nginx/index.vue index a58b9adeef5c..20250521e588 100644 --- a/frontend/src/views/website/website/nginx/index.vue +++ b/frontend/src/views/website/website/nginx/index.vue @@ -31,7 +31,7 @@ - + @@ -39,14 +39,13 @@