From e626ef7bcc7967a451bc79a39e17f2e9cafa6503 Mon Sep 17 00:00:00 2001 From: ning1875 <907974064@qq.com> Date: Wed, 13 Jan 2021 16:58:55 +0800 Subject: [PATCH] =?UTF-8?q?-=20promelog=20=E6=97=B6=E5=8C=BA=E7=94=B1utc?= =?UTF-8?q?=E6=94=B9=E4=B8=BAlocal=20-=20=E4=BD=BF=E7=94=A8makefile?= =?UTF-8?q?=E7=BC=96=E8=AF=91=20-=20=E6=B7=BB=E5=8A=A0Dockerfile=20-=20?= =?UTF-8?q?=E5=AE=8C=E5=96=84readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 25 +++++++++++ README.md | 102 +++++++++++++++++++++++++++++--------------- VERSION | 1 + go.mod | 3 +- main.go | 63 ++++++++++++++++++++------- makefile | 34 +++++++++++++++ stree-index.service | 19 +++++++++ 7 files changed, 197 insertions(+), 50 deletions(-) create mode 100644 Dockerfile create mode 100644 VERSION create mode 100644 makefile create mode 100644 stree-index.service diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..16d3d76 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.14-alpine as builder +WORKDIR /usr/src/app +ENV GOPROXY=https://goproxy.cn +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ + apk add --no-cache upx ca-certificates tzdata +COPY ./go.mod ./ +COPY ./go.sum ./ +RUN go mod download +COPY . . + +RUN BUILDUSER=`whoami`@`hostname` ;\ + BUILDDATE=`date +'%Y-%m-%d %H:%M:%S'` ;\ + GITREVISION=`git rev-parse HEAD` ;\ + GITVERSION=`cat VERSION` ;\ + GITBRANCH=`git symbolic-ref --short -q HEAD` ;\ + LDFLAGES=" -X 'github.com/prometheus/common/version.BuildUser=${BUILDUSER}' -X 'github.com/prometheus/common/version.BuildDate=${BUILDDATE}' -X 'github.com/prometheus/common/version.Revision=${GITREVISION}' -X 'github.com/prometheus/common/version.Version=${GITVERSION}' -X 'github.com/prometheus/common/version.Branch=${GITBRANCH}' ";\ + echo ${LDFLAGES} ;\ + CGO_ENABLED=0 go build -o server -ldflags "${LDFLAGES}" +FROM busybox as runner +COPY --from=builder /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /usr/src/app/server /opt/app/stree-index +ENTRYPOINT [ "/opt/app/stree-index" ] +CMD [ "--config.file=/etc/stree-index.yml"] + diff --git a/README.md b/README.md index e642d9a..69e6c1d 100644 --- a/README.md +++ b/README.md @@ -141,15 +141,15 @@ type HeadIndexReader struct { - db中所有的字段出timestamp外都可以用来构建索引，而后能所有的字段都可以被用作查询条件 举例 ```python - req_data = { - 'resource_type': 'elb', - 'use_index': True, - 'labels': [ - # 查询 group 不等于CBS，name ，正则匹配.*0dff.*，stree-app等于collecter的elb资源列表 - {'key': 'group', 'value': 'CBS', 'type': 2}, - {'key': 'name', 'value': '.*0dff.*', 'type': 3}, - {'key': 'stree-app', 'value': 'collecter', 'type': 1}] - } +req_data = { + 'resource_type': 'elb', + 'use_index': True, + 'labels': [ + # 查询 group 不等于CBS，name ，正则匹配.*0dff.*，stree-app等于collecter的elb资源列表 + {'key': 'group', 'value': 'CBS', 'type': 2}, + {'key': 'name', 'value': '.*0dff.*', 'type': 3}, + {'key': 'stree-app', 'value': 'collecter', 'type': 1}] +} ``` ### 按key查询分布情况的实现 ![image.png](/img/bVcHcIS) @@ -192,45 +192,80 @@ def query_dis(): ``` # 使用 +## step 1 准备工作 +> 准备mysql和redis，并修改配置文件对应字段 ## 创建表 -根据scripts/db_schema.sql 建表 -### 资源数据表 + +> 根据scripts/db_schema.sql 建表 + +> 资源数据表 - ecs 云服务器 - elb 云负载均衡器 - rds 云关系型数据库 - dcs 云缓存 - 对应表名为 `service_tree_ecs` `service_tree_elb` `service_tree_rds` `service_tree_dcs` -### ecs 云服务器规格表 `service_tree_cloud_instance_type` -### 树结构path表 + +> ecs 云服务器规格表 `service_tree_cloud_instance_type` +> 树结构path表 +> ## 灌入数据 - 资源数据可以由同步得来，自行实现即可 -- 各个资源表中数据tags字段为json类型，切必须包含stree-index.yml的服务树tag - ```yaml - # g.p.a模型key对应table中json字段名称 - tree_info: - name_g: group - name_p: stree-project - name_a: stree-app - ``` +- 各个资源表中数据tags字段为json类型，切必须包含stree-index.yml的服务树tag + +```yaml +# g.p.a模型key对应table中json字段名称 +tree_info: + name_g: group + name_p: stree-project + name_a: stree-app +``` + ![image.png](/img/bVcHcKn) - ecs 云服务器规格表 可以由`scripts/instance_type_insert.sh`灌入，其中包含华为和aws的大部分规格数据 -## 安装stree-index -```shell -git clone https://github.com/ning1875/stree-index.git -cd dynamic-sharding/pkg/ && go build -o stree-index main.go + +## step 2 编译或下载 +> 直接下载 +```shell script +wget https://github.com/ning1875/stree-index/releases/download/v1.0/stree-index-1.0.linux-amd64.tar.gz ``` -### 补充stree-index.yml 中db，redis等信息 -### 启动服务 -```c +> 自行编译 + +```shell script +git clone https://github.com/ning1875/stree-index.git +cd stree-index && make +``` +## step 3 补充信息 +> 补充stree-index.yml 中db，redis等信息 + +## step 4 启动服务 +> 直接启动 +```shell script ./stree-index --config.file=stree-index.yml ``` -### stree-index会自动根据资源表中服务树tag构建服务树 -```c -查询path表应该有数据 +> 使用systemd启动 +```shell script +/bin/cp -f stree-index.service /etc/systemd/system/ +/bin/cp -f stree-index /bin/ +mkdir -pv /etc/stree-index +/bin/cp -f stree-index.yml /etc/stree-index/ +systemctl enable /etc/systemd/system/stree-index.service +systemctl start stree-index.service + +``` +> 观察日志 +```shell script +tail -f /var/log/messages |grep stree-index +``` +## step 5 stree-index会自动根据资源表中服务树tag构建服务树 +> 查询path表应该有数据 + +```shell script select * from service_tree_path_tree limit 20; ``` + +# 查询 ## 使用stree-index ### 查询接口数据结构 ``` @@ -276,10 +311,10 @@ select * from service_tree_path_tree limit 20; - 2：not_eq 不等于 : key!=value - 3：reg 正则匹配 : key=~value - 4：not_reg 正则非匹配 : key!~value -~~- 对比 : key> value~~ +~~- 对比 : key> value~~ (暂时没支持) ### 查询条件自由组合 -`labels`可传入多个key和value组合，可自由组合不同kv查询 +- `labels`可传入多个key和value组合，可自由组合不同kv查询 ## 运维stree-index ### 监控 @@ -297,5 +332,4 @@ stree-index 会打点，使用prometheus采集查看即可 static_configs: - targets: - $stree-index:9393 - ``` \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..9f8e9b6 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0 \ No newline at end of file diff --git a/go.mod b/go.mod index ea9cd3f..2f50006 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/jinzhu/gorm v1.9.15 github.com/mcuadros/go-gin-prometheus v0.1.0 - github.com/oklog/run v1.1.0 // indirect + github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.5.0 github.com/prometheus/common v0.10.0 @@ -20,6 +20,7 @@ require ( github.com/spf13/viper v1.7.0 github.com/zsais/go-gin-prometheus v0.1.0 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect gopkg.in/redis.v4 v4.2.4 gopkg.in/yaml.v2 v2.3.0 diff --git a/main.go b/main.go index c46ab57..0db5a32 100644 --- a/main.go +++ b/main.go @@ -1,38 +1,71 @@ package main import ( + "context" "os" "os/signal" - "context" + "path/filepath" "syscall" + "time" - "gopkg.in/alecthomas/kingpin.v2" - "github.com/oklog/run" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/oklog/run" "github.com/prometheus/common/promlog" - "github.com/prometheus/common/promlog/flag" + promlogflag "github.com/prometheus/common/promlog/flag" "github.com/prometheus/common/version" + "gopkg.in/alecthomas/kingpin.v2" "stree-index/pkg" "stree-index/pkg/config" - "stree-index/pkg/web" - "stree-index/pkg/statistics" "stree-index/pkg/mem-index" + "stree-index/pkg/statistics" + "stree-index/pkg/web" ) func main() { var ( - configFile = kingpin.Flag("config.file", "stree-index configuration file path.").Default("stree-index.yml").String() + app = kingpin.New(filepath.Base(os.Args[0]), "The stree-index") + //configFile = kingpin.Flag("config.file", "docker-mon configuration file path.").Default("docker-mon.yml").String() + configFile = app.Flag("config.file", "docker-mon configuration file path.").Default("stree-index.yml").String() ) - - // init logger - promlogConfig := &promlog.Config{} - flag.AddFlags(kingpin.CommandLine, promlogConfig) - kingpin.Version(version.Print("stree-index")) - kingpin.HelpFlag.Short('h') - kingpin.Parse() - logger := promlog.New(promlogConfig) + promlogConfig := promlog.Config{} + + app.Version(version.Print("stree-index")) + app.HelpFlag.Short('h') + promlogflag.AddFlags(app, &promlogConfig) + kingpin.MustParse(app.Parse(os.Args[1:])) + + var logger log.Logger + logger = func(config *promlog.Config) log.Logger { + var ( + l log.Logger + le level.Option + ) + if config.Format.String() == "logfmt" { + l = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) + } else { + l = log.NewJSONLogger(log.NewSyncWriter(os.Stderr)) + } + + switch config.Level.String() { + case "debug": + le = level.AllowDebug() + case "info": + le = level.AllowInfo() + case "warn": + le = level.AllowWarn() + case "error": + le = level.AllowError() + } + l = level.NewFilter(l, le) + l = log.With(l, "ts", log.TimestampFormat( + func() time.Time { return time.Now().Local() }, + "2006-01-02T15:04:05.000Z07:00", + ), "caller", log.DefaultCaller) + return l + }(&promlogConfig) // new ctxall ctxAll, cancelAll := context.WithCancel(context.Background()) diff --git a/makefile b/makefile new file mode 100644 index 0000000..15199a2 --- /dev/null +++ b/makefile @@ -0,0 +1,34 @@ +GOCMD=go +GOBUILD=${GOCMD} build +GOCLEAN=${GOCMD} clean +GOTEST=${GOCMD} test +GOGET=${GOCMD} get + +DATE= `date +%FT%T%z` + + +BINARY_NAME="`pwd |awk -F '/' '{print $NF}'`" +BINARY_LINUX=${BINARY_NAME}_linux +BUILDUSER=`whoami`@`hostname` +BUILDDATE=`date +'%Y-%m-%d %H:%M:%S'` +GITREVISION=`git rev-parse HEAD` +GITVERSION=`cat VERSION` +GITBRANCH=`git symbolic-ref --short -q HEAD` +LDFLAGES=" -X 'github.com/prometheus/common/version.BuildUser=${BUILDUSER}' -X 'github.com/prometheus/common/version.BuildDate=${BUILDDATE}' -X 'github.com/prometheus/common/version.Revision=${GITREVISION}' -X 'github.com/prometheus/common/version.Version=${GITVERSION}' -X 'github.com/prometheus/common/version.Branch=${GITBRANCH}' " + +all: build +build: + ${GOBUILD} -v -ldflags ${LDFLAGES} -o ${BINARY_NAME} +test: + ${GOTEST} -v ./... +clean: + ${GOCLEAN} + rm -f ${BINARY_NAME} + rm -f ${BINARY_LINUX} +run: + ${GOBUILD} -o ${BINARY_NAME} -v ./... + ./${BINARY_NAME} + + +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 ${GOBUILD} -ldflags ${LDFLAGES} -o ${BINARY_LINUX} -v \ No newline at end of file diff --git a/stree-index.service b/stree-index.service new file mode 100644 index 0000000..d688c05 --- /dev/null +++ b/stree-index.service @@ -0,0 +1,19 @@ +[Unit] +Description="ning1875 opensource service tree" +Documentation=https://github.com/ning1875/stree-index +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/stree-index -f /etc/stree-index/stree-index.yml +Restart=on-failure +RestartSecs=10s +SuccessExitStatus=0 +LimitNOFILE=3000000 +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=stree-index + +[Install] +WantedBy=multi-user.target +