From 5fb227c293170a3a9a2d1552568acbedab30fdc5 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Fri, 30 Oct 2020 00:51:20 +1100 Subject: [PATCH 01/47] [query] Add additional parser tests (#2811) --- src/query/parser/promql/parse_test.go | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/query/parser/promql/parse_test.go b/src/query/parser/promql/parse_test.go index 0dd831505b..37ec76a26d 100644 --- a/src/query/parser/promql/parse_test.go +++ b/src/query/parser/promql/parse_test.go @@ -178,6 +178,35 @@ func TestAggregateParses(t *testing.T) { } } +var aggregationWithTagListTests = []string{ + // different number of tags + "sum(up) by (t1,)", + "sum(up) by (t1,t2)", + "sum(up) without (t1)", + "sum(up) without (t1, t2, t3)", + + // trailing comma in tag list + "sum(up) by (t1,)", + "sum(up) without (t1, t2,)", + + // alternative form + "sum by (t) (up)", + "sum by (t,) (up)", + "sum without (t) (up)", + "sum without (t,) (up)", +} + +func TestAggregationWithTagListDoesNotError(t *testing.T) { + for _, q := range aggregationWithTagListTests { + t.Run(q, func(t *testing.T) { + p, err := Parse(q, time.Second, models.NewTagOptions(), NewParseOptions()) + require.NoError(t, err) + _, _, err = p.DAG() + require.NoError(t, err) + }) + } +} + var linearParseTests = []struct { q string expectedType string @@ -353,6 +382,15 @@ var binaryParseTests = []struct { {"up and up", functions.FetchType, functions.FetchType, binary.AndType}, {"up or up", functions.FetchType, functions.FetchType, binary.OrType}, {"up unless up", functions.FetchType, functions.FetchType, binary.UnlessType}, + + // Various spacing + {"up/ up", functions.FetchType, functions.FetchType, binary.DivType}, + {"up-up", functions.FetchType, functions.FetchType, binary.MinusType}, + {"10 -up", scalar.ScalarType, functions.FetchType, binary.MinusType}, + {"up*10", functions.FetchType, scalar.ScalarType, binary.MultiplyType}, + {"up!=10", functions.FetchType, scalar.ScalarType, binary.NotEqType}, + {"10 = 10", functions.FetchType, scalar.ScalarType, binary.GreaterEqType}, } func TestBinaryParses(t *testing.T) { From cb2d5f7ad7b426fa48315b2a946d6127cf61774e Mon Sep 17 00:00:00 2001 From: Bo Du Date: Thu, 29 Oct 2020 12:09:11 -0400 Subject: [PATCH 02/47] [dbnode] Move eviction logic up before ns loop (#2812) --- src/dbnode/storage/bootstrap/bootstrapper/peers/source.go | 6 +++--- src/dbnode/storage/bootstrap/cache.go | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go index 80ad187312..0ab602644e 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go @@ -171,6 +171,9 @@ func (s *peersSource) Read( zap.Duration("took", s.nowFn().Sub(start))) span.LogEvent("bootstrap_data_done") + // NB(bodu): We need to evict the info file cache before reading index data since we've + // maybe fetched blocks from peers so the cached info file state is now stale. + cache.Evict() start = s.nowFn() s.log.Info("bootstrapping index metadata start") span.LogEvent("bootstrap_index_start") @@ -185,9 +188,6 @@ func (s *peersSource) Read( continue } - // NB(bodu): We need to evict the info file cache before reading index data since we've - // maybe fetched blocks from peers so the cached info file state is now stale. - cache.Evict() r, err := s.readIndex(md, namespace.IndexRunOptions.ShardTimeRanges, span, diff --git a/src/dbnode/storage/bootstrap/cache.go b/src/dbnode/storage/bootstrap/cache.go index 77c8e20633..50ebad8349 100644 --- a/src/dbnode/storage/bootstrap/cache.go +++ b/src/dbnode/storage/bootstrap/cache.go @@ -103,7 +103,12 @@ func (c *cache) ReadInfoFiles() InfoFilesByNamespace { func (c *cache) populateInfoFilesByNamespaceWithLock() { for _, finder := range c.namespaceDetails { - result := make(InfoFileResultsPerShard, len(finder.Shards)) + // NB(bodu): It is okay to reuse the info files by ns results per shard here + // as the shards were set in the cache ctor and do not change per invocation. + result, ok := c.infoFilesByNamespace[finder.Namespace] + if !ok { + result = make(InfoFileResultsPerShard, len(finder.Shards)) + } for _, shard := range finder.Shards { result[shard] = fs.ReadInfoFiles(c.fsOpts.FilePathPrefix(), finder.Namespace.ID(), shard, c.fsOpts.InfoReaderBufferSize(), c.fsOpts.DecodingOptions(), From 6cd01489eba05ae5f1df9441e639cf9e573ab6d6 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 29 Oct 2020 15:31:11 -0400 Subject: [PATCH 03/47] [dbnode] Update return unfulfilled for corrupt commit log files default (#2807) --- config/m3db/clustered-etcd/generated.yaml | 3 --- config/m3db/clustered-etcd/m3dbnode.libsonnet | 5 ----- config/m3db/local-etcd/generated.yaml | 3 --- config/m3db/local-etcd/m3dbnode.libsonnet | 5 ----- kube/bundle.yaml | 4 ---- kube/m3dbnode-configmap.yaml | 4 ---- kube/terraform/main.tf | 2 +- scripts/development/m3_stack/m3dbnode.yml | 4 ---- .../dedicated_etcd_embedded_coordinator/m3dbnode.yml | 4 ---- .../multi_cluster_write/m3dbnode-cluster-a.yml | 4 ---- .../multi_cluster_write/m3dbnode-cluster-b.yml | 4 ---- scripts/docker-integration-tests/repair/m3dbnode.yml | 4 ---- .../repair_and_replication/m3dbnode-cluster-a.yml | 4 ---- .../repair_and_replication/m3dbnode-cluster-b.yml | 4 ---- .../replication/m3dbnode-cluster-a.yml | 4 ---- .../replication/m3dbnode-cluster-b.yml | 4 ---- .../operational_guide/availability_consistency_durability.md | 2 +- .../tools/dtest/docker/harness/resources/config/m3dbnode.yml | 4 ---- src/dbnode/config/m3dbnode-all-config.yml | 1 + src/dbnode/config/m3dbnode-cluster-template.yml | 4 ---- src/dbnode/config/m3dbnode-local-etcd-proto.yml | 4 ---- src/dbnode/config/m3dbnode-local-etcd.yml | 4 ---- .../storage/bootstrap/bootstrapper/commitlog/options.go | 2 +- .../bootstrap/bootstrapper/commitlog/source_prop_test.go | 3 ++- 24 files changed, 6 insertions(+), 80 deletions(-) diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml index 646ee745a8..f62e7ada30 100644 --- a/config/m3db/clustered-etcd/generated.yaml +++ b/config/m3db/clustered-etcd/generated.yaml @@ -19,9 +19,6 @@ "tagOptions": "idScheme": "quoted" "db": - "bootstrap": - "commitlog": - "returnUnfulfilledForCorruptCommitLogFiles": false "cache": "postingsList": "size": 262144 diff --git a/config/m3db/clustered-etcd/m3dbnode.libsonnet b/config/m3db/clustered-etcd/m3dbnode.libsonnet index a3439439b5..50bb6fa1ed 100644 --- a/config/m3db/clustered-etcd/m3dbnode.libsonnet +++ b/config/m3db/clustered-etcd/m3dbnode.libsonnet @@ -103,11 +103,6 @@ function(cluster, coordinator={}, db={}) { "writeNewSeriesAsync": true, "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", - "bootstrap": { - "commitlog": { - "returnUnfulfilledForCorruptCommitLogFiles": false - } - }, "cache": { "series": { "policy": "lru" diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index d98d832cee..cc5b10c10c 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -19,9 +19,6 @@ "tagOptions": "idScheme": "quoted" "db": - "bootstrap": - "commitlog": - "returnUnfulfilledForCorruptCommitLogFiles": false "cache": "postingsList": "size": 262144 diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet index a04ab6fb3a..fc6e5c5651 100644 --- a/config/m3db/local-etcd/m3dbnode.libsonnet +++ b/config/m3db/local-etcd/m3dbnode.libsonnet @@ -62,11 +62,6 @@ function(coordinator={}, db={}) { "writeNewSeriesAsync": true, "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", - "bootstrap": { - "commitlog": { - "returnUnfulfilledForCorruptCommitLogFiles": false - } - }, "cache": { "series": { "policy": "lru" diff --git a/kube/bundle.yaml b/kube/bundle.yaml index 348bf32c24..8e99d0b72e 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -166,10 +166,6 @@ data: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - commitlog: flushMaxBytes: 524288 flushEvery: 1s diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 4221fec07c..f51032e5c9 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -56,10 +56,6 @@ data: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - commitlog: flushMaxBytes: 524288 flushEvery: 1s diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 324f91b853..675805edda 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n filesystem:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 54a55ba6f2..4243c971c1 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index a0689a4f27..d94b0d3993 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -48,10 +48,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 0d1a4c4c67..5527f12830 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -59,10 +59,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index ba15ee5ebc..edbfcc298b 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index ed221416b3..b6f5cb5436 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index 01613fb5f8..93dedab49d 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index 29edf3d35a..b2c5b2c650 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index ee44d87669..9ba35ef331 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index cdd084aa3d..c3097f8df8 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -38,10 +38,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/site/content/docs/operational_guide/availability_consistency_durability.md b/site/content/docs/operational_guide/availability_consistency_durability.md index a23e57d286..873ed4c6aa 100644 --- a/site/content/docs/operational_guide/availability_consistency_durability.md +++ b/site/content/docs/operational_guide/availability_consistency_durability.md @@ -68,7 +68,7 @@ This value can be set much lower than the default value for workloads in which a ### Ignoring Corrupt Commitlogs on Bootstrap If M3DB is shut down gracefully (i.e via SIGTERM), it will ensure that all pending writes are flushed to the commitlog on disk before the process exists. -However, in situations where the process crashed/exited unexpectedly or the node itself experienced a sudden failure, the tail end of the commitlog may be corrupt. +However, in situations where SIGKILL is used, the process exited unexpectedly or the node itself experienced a sudden failure, the tail end of the commitlog may be corrupt. In such situations, M3DB will read as much of the commitlog as possible in an attempt to recover the maximum amount of data. However, it then needs to make a decision: it can either **(a)** come up successfully and tolerate an ostensibly minor amount of data or loss, or **(b)** attempt to stream the missing data from its peers. This behavior is controlled by the following default configuration: diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 9c2b6f1dec..1f9ab9fed6 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -55,10 +55,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index bc0ff7c742..0849aa4cd9 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -105,6 +105,7 @@ db: bootstrap: commitlog: + # Whether tail end of corrupted commit logs cause an error on bootstrap. returnUnfulfilledForCorruptCommitLogFiles: false cache: diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index 085b9ee694..d5cddf3c5a 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -76,10 +76,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 82f877c9b6..0d34b5f53a 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -55,10 +55,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 9c2b6f1dec..1f9ab9fed6 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -55,10 +55,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - commitlog: - returnUnfulfilledForCorruptCommitLogFiles: false - cache: series: policy: lru diff --git a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/options.go b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/options.go index 39f098ea46..9a559213fc 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/options.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/options.go @@ -34,7 +34,7 @@ const ( // DefaultReturnUnfulfilledForCorruptCommitLogFiles is the default // value for whether to return unfulfilled when encountering corrupt // commit log files. - DefaultReturnUnfulfilledForCorruptCommitLogFiles = true + DefaultReturnUnfulfilledForCorruptCommitLogFiles = false ) var ( diff --git a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go index adc957cdcd..6dea040bbe 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go @@ -134,7 +134,8 @@ func TestCommitLogSourcePropCorrectlyBootstrapsFromCommitlog(t *testing.T) { SetStrategy(commitlog.StrategyWriteBehind). SetFlushInterval(time.Millisecond). SetClockOptions(testCommitlogOpts.ClockOptions().SetNowFn(nowFn)) - bootstrapOpts = testDefaultOpts.SetCommitLogOptions(commitLogOpts) + bootstrapOpts = testDefaultOpts.SetCommitLogOptions(commitLogOpts). + SetReturnUnfulfilledForCorruptCommitLogFiles(true) start = input.currentTime.Truncate(blockSize) ) From 658175d069479d081ff00aa69dcbb1faedb7eb89 Mon Sep 17 00:00:00 2001 From: Matt Way Date: Thu, 29 Oct 2020 17:26:15 -0400 Subject: [PATCH 04/47] [metrics/rules] Add test to assert that mixed-mode keepOriginal uses keepOriginal=true (#2819) --- src/metrics/rules/active_ruleset_test.go | 131 +++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/src/metrics/rules/active_ruleset_test.go b/src/metrics/rules/active_ruleset_test.go index a3f54d4276..809e36b893 100644 --- a/src/metrics/rules/active_ruleset_test.go +++ b/src/metrics/rules/active_ruleset_test.go @@ -567,6 +567,34 @@ func TestActiveRuleSetForwardMatchWithMappingRules(t *testing.T) { } } +func TestActiveRuleSetForwardMatchWithAnyKeepOriginal(t *testing.T) { + inputs := []testMatchInput{ + { + id: "rtagName1=rtagValue1", + matchFrom: 25000, + matchTo: 25001, + keepOriginal: true, + }, + } + + as := newActiveRuleSet( + 0, + nil, + testKeepOriginalRollupRules(t), + testTagsFilterOptions(), + mockNewID, + nil, + ) + + for i, input := range inputs { + t.Run(fmt.Sprintf("input %d", i), func(t *testing.T) { + res := as.ForwardMatch(b(input.id), input.matchFrom, input.matchTo) + require.Equal(t, res.keepOriginal, input.keepOriginal) + require.Equal(t, 3, res.NumNewRollupIDs()) + }) + } +} + func TestActiveRuleSetForwardMatchWithRollupRules(t *testing.T) { inputs := []testMatchInput{ { @@ -3605,6 +3633,109 @@ func testMappingRules(t *testing.T) []*mappingRule { mappingRule10, mappingRule11} } +func testKeepOriginalRollupRules(t *testing.T) []*rollupRule { + filter, err := filters.NewTagsFilter( + filters.TagFilterValueMap{ + "rtagName1": filters.FilterValue{Pattern: "rtagValue1"}, + }, + filters.Conjunction, + testTagsFilterOptions(), + ) + require.NoError(t, err) + + rollupRule1 := &rollupRule{ + uuid: "rollupRule1", + snapshots: []*rollupRuleSnapshot{ + &rollupRuleSnapshot{ + name: "rollupRule1.snapshot", + tombstoned: false, + cutoverNanos: 0, + filter: filter, + keepOriginal: false, + targets: []rollupTarget{ + { + Pipeline: pipeline.NewPipeline([]pipeline.OpUnion{ + { + Type: pipeline.RollupOpType, + Rollup: pipeline.RollupOp{ + NewName: b("rName1"), + Tags: bs("rtagName1"), + AggregationID: aggregation.DefaultID, + }, + }, + }), + StoragePolicies: policy.StoragePolicies{ + policy.NewStoragePolicy(10*time.Second, xtime.Second, 24*time.Hour), + }, + }, + }, + }, + }, + } + + rollupRule2 := &rollupRule{ + uuid: "rollupRule2", + snapshots: []*rollupRuleSnapshot{ + &rollupRuleSnapshot{ + name: "rollupRule2.snapshot", + tombstoned: false, + cutoverNanos: 0, + filter: filter, + keepOriginal: true, + targets: []rollupTarget{ + { + Pipeline: pipeline.NewPipeline([]pipeline.OpUnion{ + { + Type: pipeline.RollupOpType, + Rollup: pipeline.RollupOp{ + NewName: b("rName2"), + Tags: bs("rtagName1"), + AggregationID: aggregation.DefaultID, + }, + }, + }), + StoragePolicies: policy.StoragePolicies{ + policy.NewStoragePolicy(10*time.Second, xtime.Second, 24*time.Hour), + }, + }, + }, + }, + }, + } + + rollupRule3 := &rollupRule{ + uuid: "rollupRule3", + snapshots: []*rollupRuleSnapshot{ + &rollupRuleSnapshot{ + name: "rollupRule3.snapshot", + tombstoned: false, + cutoverNanos: 0, + filter: filter, + keepOriginal: true, + targets: []rollupTarget{ + { + Pipeline: pipeline.NewPipeline([]pipeline.OpUnion{ + { + Type: pipeline.RollupOpType, + Rollup: pipeline.RollupOp{ + NewName: b("rName3"), + Tags: bs("rtagName1"), + AggregationID: aggregation.DefaultID, + }, + }, + }), + StoragePolicies: policy.StoragePolicies{ + policy.NewStoragePolicy(10*time.Second, xtime.Second, 24*time.Hour), + }, + }, + }, + }, + }, + } + + return []*rollupRule{rollupRule1, rollupRule2, rollupRule3} +} + func testRollupRules(t *testing.T) []*rollupRule { filter1, err := filters.NewTagsFilter( filters.TagFilterValueMap{ From 8d430761c4743ad00ed625d9fa57c1f64fcc6933 Mon Sep 17 00:00:00 2001 From: Ryan Hall Date: Thu, 29 Oct 2020 16:34:54 -0700 Subject: [PATCH 05/47] AddToReset config to change Add transforms to Reset transforms (#2817) This is a temporary option to allow seamlessly rolling out Add -> Reset transforms. With this we can deploy the new aggregator with the option without experiencing another degradation in aggregated counters. Without the change we'd need to deploy the new code and then deploy the new rules. The deploy would result in degraded aggregated counters without this config. --- src/aggregator/aggregator/elem_base.go | 11 +++++++++- src/aggregator/aggregator/elem_base_test.go | 22 +++++++++++++++++++ src/aggregator/aggregator/options.go | 19 ++++++++++++++++ .../m3aggregator/config/aggregator.go | 6 ++++- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/aggregator/aggregator/elem_base.go b/src/aggregator/aggregator/elem_base.go index eeff7cc5b2..973e223345 100644 --- a/src/aggregator/aggregator/elem_base.go +++ b/src/aggregator/aggregator/elem_base.go @@ -37,9 +37,9 @@ import ( "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/metrics/transformation" "github.com/m3db/m3/src/x/pool" - "go.uber.org/zap" "github.com/willf/bitset" + "go.uber.org/zap" ) const ( @@ -166,6 +166,7 @@ type elemBase struct { idPrefixSuffixType IDPrefixSuffixType writeForwardedMetricFn writeForwardedMetricFn onForwardedAggregationWrittenFn onForwardedAggregationDoneFn + addToReset bool // Mutable states. tombstoned bool @@ -179,6 +180,7 @@ func newElemBase(opts Options) elemBase { opts: opts, aggTypesOpts: opts.AggregationTypesOptions(), aggOpts: raggregation.NewOptions(opts.InstrumentOptions()), + addToReset: opts.AddToReset(), } } @@ -198,6 +200,13 @@ func (e *elemBase) resetSetData( l.Error("error parsing pipeline", zap.Error(err)) return err } + if e.addToReset { + for i := range parsed.Transformations { + if parsed.Transformations[i].Type() == transformation.Add { + parsed.Transformations[i], _ = transformation.Reset.NewOp() + } + } + } e.id = id e.sp = sp e.aggTypes = aggTypes diff --git a/src/aggregator/aggregator/elem_base_test.go b/src/aggregator/aggregator/elem_base_test.go index a929227a26..a277432a51 100644 --- a/src/aggregator/aggregator/elem_base_test.go +++ b/src/aggregator/aggregator/elem_base_test.go @@ -572,3 +572,25 @@ func TestParsePipelineTransformationDerivativeOrderTooHigh(t *testing.T) { require.Error(t, err) require.True(t, strings.Contains(err.Error(), "transformation derivative order is 2 higher than supported 1")) } + +func TestAddToRestOption(t *testing.T) { + p := applied.NewPipeline([]applied.OpUnion{ + { + Type: pipeline.TransformationOpType, + Transformation: pipeline.TransformationOp{Type: transformation.Add}, + }, + { + Type: pipeline.TransformationOpType, + Transformation: pipeline.TransformationOp{Type: transformation.Increase}, + }, + }) + e := newElemBase(NewOptions().SetAddToReset(false)) + e.resetSetData(testCounterID, testStoragePolicy, testAggregationTypesExpensive, false, p, 3, WithPrefixWithSuffix) + require.Equal(t, transformation.Add, e.parsedPipeline.Transformations[0].Type()) + require.Equal(t, transformation.Increase, e.parsedPipeline.Transformations[1].Type()) + + e = newElemBase(NewOptions().SetAddToReset(true)) + e.resetSetData(testCounterID, testStoragePolicy, testAggregationTypesExpensive, false, p, 3, WithPrefixWithSuffix) + require.Equal(t, transformation.Reset, e.parsedPipeline.Transformations[0].Type()) + require.Equal(t, transformation.Increase, e.parsedPipeline.Transformations[1].Type()) +} diff --git a/src/aggregator/aggregator/options.go b/src/aggregator/aggregator/options.go index 006801f365..b90d915e6c 100644 --- a/src/aggregator/aggregator/options.go +++ b/src/aggregator/aggregator/options.go @@ -314,6 +314,14 @@ type Options interface { // VerboseErrors returns whether to return verbose errors or not. VerboseErrors() bool + + // SetAddToReset sets the value for AddToReset. + SetAddToReset(value bool) Options + + // AddToReset changes Add transforms to Reset Transforms. + // This is a temporary option to help with the seamless rollout of changing Add transforms to Reset transforms for + // resetting aggregate counters. Once rollup rules have changed to use Reset explicitly, this can be removed. + AddToReset() bool } type options struct { @@ -354,6 +362,7 @@ type options struct { timerElemPool TimerElemPool gaugeElemPool GaugeElemPool verboseErrors bool + addToReset bool // Derived options. fullCounterPrefix []byte @@ -840,6 +849,16 @@ func (o *options) computeFullGaugePrefix() { o.fullGaugePrefix = fullGaugePrefix } +func (o *options) AddToReset() bool { + return o.addToReset +} + +func (o *options) SetAddToReset(value bool) Options { + opts := *o + opts.addToReset = value + return &opts +} + func defaultMaxAllowedForwardingDelayFn( resolution time.Duration, numForwardedTimes int, diff --git a/src/cmd/services/m3aggregator/config/aggregator.go b/src/cmd/services/m3aggregator/config/aggregator.go index 307fa69212..89da0b119e 100644 --- a/src/cmd/services/m3aggregator/config/aggregator.go +++ b/src/cmd/services/m3aggregator/config/aggregator.go @@ -173,6 +173,9 @@ type AggregatorConfiguration struct { // Pool of entries. EntryPool pool.ObjectPoolConfiguration `yaml:"entryPool"` + + // AddToReset is the yaml config for aggregator.Options.AddToReset + AddToReset bool `yaml:"addToReset"` } // InstanceIDType is the instance ID type that defines how the @@ -258,7 +261,8 @@ func (c *AggregatorConfiguration) NewAggregatorOptions( opts := aggregator.NewOptions(). SetInstrumentOptions(instrumentOpts). SetRuntimeOptionsManager(runtimeOptsManager). - SetVerboseErrors(c.VerboseErrors) + SetVerboseErrors(c.VerboseErrors). + SetAddToReset(c.AddToReset) rwOpts := serveOpts.RWOptions() if rwOpts == nil { From eb7a85b8d41a64bdf820e41ef34695f763e4f87c Mon Sep 17 00:00:00 2001 From: Matt Way Date: Thu, 29 Oct 2020 21:07:07 -0400 Subject: [PATCH 06/47] Remove old code: aggregation/quantile/tdigest (#2820) --- .../quantile/tdigest/centroids_pool.go | 49 --- .../aggregation/quantile/tdigest/doc.go | 29 -- .../aggregation/quantile/tdigest/options.go | 108 ------- .../quantile/tdigest/options_test.go | 56 ---- .../aggregation/quantile/tdigest/tdigest.go | 305 ------------------ .../quantile/tdigest/tdigest_test.go | 248 -------------- .../aggregation/quantile/tdigest/types.go | 94 ------ 7 files changed, 889 deletions(-) delete mode 100644 src/aggregator/aggregation/quantile/tdigest/centroids_pool.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/doc.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/options.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/options_test.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/tdigest.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/tdigest_test.go delete mode 100644 src/aggregator/aggregation/quantile/tdigest/types.go diff --git a/src/aggregator/aggregation/quantile/tdigest/centroids_pool.go b/src/aggregator/aggregation/quantile/tdigest/centroids_pool.go deleted file mode 100644 index 292cc3b02e..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/centroids_pool.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -import ( - "github.com/m3db/m3/src/x/pool" -) - -type centroidsPool struct { - pool pool.BucketizedObjectPool -} - -// NewCentroidsPool creates a new centroids pool. -func NewCentroidsPool(sizes []pool.Bucket, opts pool.ObjectPoolOptions) CentroidsPool { - return ¢roidsPool{pool: pool.NewBucketizedObjectPool(sizes, opts)} -} - -func (p *centroidsPool) Init() { - p.pool.Init(func(capacity int) interface{} { - return make([]Centroid, 0, capacity) - }) -} - -func (p *centroidsPool) Get(capacity int) []Centroid { - return p.pool.Get(capacity).([]Centroid) -} - -func (p *centroidsPool) Put(value []Centroid) { - value = value[:0] - p.pool.Put(value, cap(value)) -} diff --git a/src/aggregator/aggregation/quantile/tdigest/doc.go b/src/aggregator/aggregation/quantile/tdigest/doc.go deleted file mode 100644 index 9cfd86ae74..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/doc.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -/* - -Package tdigest implements the t-digest algorithm for accurate on-line accumulation -of rank-based statistics such as quantiles from "Computing Extremely Accurate Quantiles -Using t-Digests" and is based on Ted Dunning's implementation in Java: -https://github.com/tdunning/t-digest/blob/master/src/main/java/com/tdunning/math/stats/ - -*/ -package tdigest diff --git a/src/aggregator/aggregation/quantile/tdigest/options.go b/src/aggregator/aggregation/quantile/tdigest/options.go deleted file mode 100644 index 3b607d4b56..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/options.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -import ( - "errors" - "fmt" - - "github.com/m3db/m3/src/x/pool" -) - -const ( - defaultCompression = 100.0 - defaultPrecision = 0 - minPrecision = 0 - maxPrecision = 16 - minCompression = 20.0 - maxCompression = 1000.0 -) - -var ( - defaultBuckets = []pool.Bucket{ - {Capacity: 16, Count: 4096}, - } - - errInvalidCompression = fmt.Errorf("compression must be between %f and %f", minCompression, maxCompression) - errInvalidPrecision = fmt.Errorf("precision must be between %d and %d", minPrecision, maxPrecision) - errNoCentroidsPool = errors.New("no centroids pool set") -) - -type options struct { - compression float64 - precision int - centroidsPool CentroidsPool -} - -// NewOptions creates a new options. -func NewOptions() Options { - centroidsPool := NewCentroidsPool(defaultBuckets, nil) - centroidsPool.Init() - - return &options{ - compression: defaultCompression, - precision: defaultPrecision, - centroidsPool: centroidsPool, - } -} - -func (o *options) SetCompression(value float64) Options { - opts := *o - opts.compression = value - return &opts -} - -func (o *options) Compression() float64 { - return o.compression -} - -func (o *options) SetPrecision(value int) Options { - opts := *o - opts.precision = value - return &opts -} - -func (o *options) Precision() int { - return o.precision -} - -func (o *options) SetCentroidsPool(value CentroidsPool) Options { - opts := *o - opts.centroidsPool = value - return &opts -} - -func (o *options) CentroidsPool() CentroidsPool { - return o.centroidsPool -} - -func (o *options) Validate() error { - if o.compression < minCompression || o.compression > maxCompression { - return errInvalidCompression - } - if o.precision < minPrecision || o.precision > maxPrecision { - return errInvalidPrecision - } - if o.centroidsPool == nil { - return errNoCentroidsPool - } - return nil -} diff --git a/src/aggregator/aggregation/quantile/tdigest/options_test.go b/src/aggregator/aggregation/quantile/tdigest/options_test.go deleted file mode 100644 index 4889fb803f..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/options_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -var ( - testOpts = NewOptions() -) - -func TestOptionsValidateSuccess(t *testing.T) { - require.NoError(t, testOpts.Validate()) -} - -func TestOptionsValidateInvalidCompression(t *testing.T) { - opts := testOpts.SetCompression(minCompression - 1) - require.Equal(t, errInvalidCompression, opts.Validate()) - - opts = testOpts.SetCompression(maxCompression + 1) - require.Equal(t, errInvalidCompression, opts.Validate()) -} - -func TestOptionsValidateInvalidPrecision(t *testing.T) { - opts := testOpts.SetPrecision(minPrecision - 1) - require.Equal(t, errInvalidPrecision, opts.Validate()) - - opts = testOpts.SetPrecision(maxPrecision + 1) - require.Equal(t, errInvalidPrecision, opts.Validate()) -} - -func TestOptionsValidateNoCentroidsPool(t *testing.T) { - opts := testOpts.SetCentroidsPool(nil) - require.Equal(t, errNoCentroidsPool, opts.Validate()) -} diff --git a/src/aggregator/aggregation/quantile/tdigest/tdigest.go b/src/aggregator/aggregation/quantile/tdigest/tdigest.go deleted file mode 100644 index 1574bad32a..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/tdigest.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -import ( - "math" - "sort" -) - -var ( - nan = math.NaN() - positiveInfinity = math.Inf(1) - negativeInfinity = math.Inf(-1) - sentinelCentroid = Centroid{Mean: positiveInfinity, Weight: 0.0} -) - -type centroidsByMeanAsc []Centroid - -func (c centroidsByMeanAsc) Len() int { return len(c) } -func (c centroidsByMeanAsc) Less(i, j int) bool { return c[i].Mean < c[j].Mean } -func (c centroidsByMeanAsc) Swap(i, j int) { c[i], c[j] = c[j], c[i] } - -type mergeCentroidFn func( - currIndex float64, - currWeight float64, - totalWeight float64, - c Centroid, - mergeResult []Centroid, -) (float64, []Centroid) - -type tDigest struct { - compression float64 // compression factor - mergedCapacity int // merged centroid slice capacity - unmergedCapacity int // unmerged centroid slice capacity - multiplier int64 // quantile precision multiplier - mergeCentroidFn mergeCentroidFn // function to merge centroids - centroidsPool CentroidsPool // centroids pool - - closed bool // whether the t-digest is closed - merged []Centroid // merged centroids - mergedWeight float64 // total weight of merged centroids - unmerged []Centroid // unmerged centroid slice capacity - unmergedWeight float64 // total weight of unmerged centroids - minValue float64 // minimum value - maxValue float64 // maximum value -} - -// mergedCapacity computes the capacity of the merged centroid slice. -func mergedCapacity(compression float64) int { - return int(math.Ceil(math.Pi*compression + 0.5)) -} - -func unmergedCapacity(compression float64) int { - // NB: the formula is taken from tdunning's implementation by - // regressing against known sizes for sample compression values. - compression = math.Min(math.Max(20, compression), 1000) - return int(7.5 + 0.37*compression - 2e-4*compression*compression) -} - -// NewTDigest creates a new t-digest. -// TODO(xichen): add pooling for t-digests -// TODO(xichen): add metrics -func NewTDigest(opts Options) TDigest { - centroidsPool := opts.CentroidsPool() - compression := opts.Compression() - mergedCapacity := mergedCapacity(compression) - unmergedCapacity := unmergedCapacity(compression) - - var multiplier int64 - if precision := opts.Precision(); precision != 0 { - multiplier = int64(math.Pow10(precision)) - } - - d := &tDigest{ - compression: opts.Compression(), - multiplier: multiplier, - centroidsPool: centroidsPool, - mergedCapacity: mergedCapacity, - unmergedCapacity: unmergedCapacity, - } - d.mergeCentroidFn = d.mergeCentroid - - d.Reset() - return d -} - -func (d *tDigest) Merged() []Centroid { - return d.merged -} - -func (d *tDigest) Unmerged() []Centroid { - return d.unmerged -} - -func (d *tDigest) Add(value float64) { - d.add(value, 1.0) -} - -func (d *tDigest) Min() float64 { - return d.Quantile(0.0) -} - -func (d *tDigest) Max() float64 { - return d.Quantile(1.0) -} - -func (d *tDigest) Quantile(q float64) float64 { - if q < 0.0 || q > 1.0 { - return nan - } - - // compress the centroids first. - d.compress() - - // If the digest is empty, return 0. - if len(d.merged) == 0 { - return 0.0 - } - - if q == 0.0 { - return d.minValue - } - - if q == 1.0 { - return d.maxValue - } - - var ( - targetWeight = q * d.mergedWeight - currWeight = 0.0 - lowerBound = d.minValue - upperBound float64 - ) - for i, c := range d.merged { - upperBound = d.upperBound(i) - if targetWeight <= currWeight+c.Weight { - // The quantile falls within this centroid. - ratio := (targetWeight - currWeight) / c.Weight - quantile := lowerBound + ratio*(upperBound-lowerBound) - // If there is a desired precision, we truncate the quantile per the precision requirement. - if d.multiplier != 0 { - quantile = math.Trunc(quantile*float64(d.multiplier)) / float64(d.multiplier) - } - return quantile - } - currWeight += c.Weight - lowerBound = upperBound - } - - // NB(xichen): should never get here unless the centroids array are empty - // because the target weight should always be no larger than the total weight. - return nan -} - -func (d *tDigest) Merge(tdigest TDigest) { - merged := tdigest.Merged() - for _, c := range merged { - d.add(c.Mean, c.Weight) - } - - unmerged := tdigest.Unmerged() - for _, c := range unmerged { - d.add(c.Mean, c.Weight) - } -} - -func (d *tDigest) Close() { - if d.closed { - return - } - d.closed = true - d.centroidsPool.Put(d.merged) - d.centroidsPool.Put(d.unmerged) -} - -func (d *tDigest) Reset() { - d.closed = false - d.merged = d.centroidsPool.Get(d.mergedCapacity) - d.mergedWeight = 0.0 - d.unmerged = d.centroidsPool.Get(d.unmergedCapacity) - d.unmergedWeight = 0.0 - d.minValue = positiveInfinity - d.maxValue = negativeInfinity -} - -// compress merges unmerged centroids and merged centroids. -func (d *tDigest) compress() { - if len(d.unmerged) == 0 { - return - } - - sort.Sort(centroidsByMeanAsc(d.unmerged)) - - var ( - totalWeight = d.mergedWeight + d.unmergedWeight - currWeight = 0.0 - currIndex = 0.0 - mergedIndex = 0 - unmergedIndex = 0 - mergeResult = d.centroidsPool.Get(len(d.merged) + len(d.unmerged)) - ) - - for mergedIndex < len(d.merged) || unmergedIndex < len(d.unmerged) { - currUnmerged := sentinelCentroid - if unmergedIndex < len(d.unmerged) { - currUnmerged = d.unmerged[unmergedIndex] - } - - currMerged := sentinelCentroid - if mergedIndex < len(d.merged) { - currMerged = d.merged[mergedIndex] - } - - if currUnmerged.Mean < currMerged.Mean { - currIndex, mergeResult = d.mergeCentroidFn(currIndex, currWeight, totalWeight, currUnmerged, mergeResult) - currWeight += currUnmerged.Weight - unmergedIndex++ - } else { - currIndex, mergeResult = d.mergeCentroidFn(currIndex, currWeight, totalWeight, currMerged, mergeResult) - currWeight += currMerged.Weight - mergedIndex++ - } - } - - d.centroidsPool.Put(d.merged) - d.merged = mergeResult - d.mergedWeight = totalWeight - d.unmerged = d.unmerged[:0] - d.unmergedWeight = 0.0 -} - -// mergeCentroid merges a centroid into the list of merged centroids. -func (d *tDigest) mergeCentroid( - currIndex float64, - currWeight float64, - totalWeight float64, - c Centroid, - mergeResult []Centroid, -) (float64, []Centroid) { - nextIndex := d.nextIndex((currWeight + c.Weight) / totalWeight) - if nextIndex-currIndex > 1 || len(mergeResult) == 0 { - // This is the first centroid added, or the next index is too far away from the current index. - mergeResult = d.appendCentroid(mergeResult, c) - return d.nextIndex(currWeight / totalWeight), mergeResult - } - - // The new centroid falls within the range of the current centroid. - numResults := len(mergeResult) - mergeResult[numResults-1].Weight += c.Weight - mergeResult[numResults-1].Mean += (c.Mean - mergeResult[numResults-1].Mean) * c.Weight / mergeResult[numResults-1].Weight - return currIndex, mergeResult -} - -// nextIndex estimates the index of the next centroid. -func (d *tDigest) nextIndex(quantile float64) float64 { - return d.compression * (math.Asin(2*quantile-1)/math.Pi + 0.5) -} - -// add adds a weighted value. -func (d *tDigest) add(value float64, weight float64) { - if len(d.unmerged) == d.unmergedCapacity { - d.compress() - } - d.minValue = math.Min(d.minValue, value) - d.maxValue = math.Max(d.maxValue, value) - d.unmerged = d.appendCentroid(d.unmerged, Centroid{Mean: value, Weight: weight}) - d.unmergedWeight += weight -} - -// upperBound returns the upper bound for computing quantiles given the centroid index. -// d.merged is guaranteed to have at least one centroid when upperBound is called. -func (d *tDigest) upperBound(index int) float64 { - if index == len(d.merged)-1 { - return d.maxValue - } - return (d.merged[index].Mean + d.merged[index+1].Mean) / 2.0 -} - -func (d *tDigest) appendCentroid(centroids []Centroid, c Centroid) []Centroid { - if len(centroids) == cap(centroids) { - newCentroids := d.centroidsPool.Get(2 * len(centroids)) - newCentroids = append(newCentroids, centroids...) - d.centroidsPool.Put(centroids) - centroids = newCentroids - } - return append(centroids, c) -} diff --git a/src/aggregator/aggregation/quantile/tdigest/tdigest_test.go b/src/aggregator/aggregation/quantile/tdigest/tdigest_test.go deleted file mode 100644 index 591e236037..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/tdigest_test.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -import ( - "math" - "math/rand" - "testing" - - "github.com/m3db/m3/src/x/pool" - - "github.com/stretchr/testify/require" -) - -var ( - testQuantiles = []float64{0.5, 0.95, 0.99} -) - -func testTDigestOptions() Options { - return NewOptions() -} - -func TestEmptyTDigest(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts) - require.Equal(t, 0.0, d.Min()) - require.Equal(t, 0.0, d.Max()) - for _, q := range testQuantiles { - require.Equal(t, 0.0, d.Quantile(q)) - } -} - -func TestTDigestWithOutOfBoundsQuantile(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts) - require.True(t, math.IsNaN(d.Quantile(-1.0))) - require.True(t, math.IsNaN(d.Quantile(10.0))) -} - -func TestTDigestWithOneValue(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts) - d.Add(100.0) - require.Equal(t, 100.0, d.Min()) - require.Equal(t, 100.0, d.Max()) - for _, q := range testQuantiles { - require.Equal(t, 100.0, d.Quantile(q)) - } -} - -func TestTDigestWithThreeValues(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts) - for _, val := range []float64{100.0, 200.0, 300.0} { - d.Add(val) - } - - require.Equal(t, 100.0, d.Min()) - require.Equal(t, 300.0, d.Max()) - expected := []float64{200.0, 300.0, 300.0} - for i, q := range testQuantiles { - require.InEpsilon(t, expected[i], d.Quantile(q), 0.025) - } -} - -func TestTDigestWithIncreasingValues(t *testing.T) { - numSamples := 100000 - opts := testTDigestOptions() - d := NewTDigest(opts) - for i := 0; i < numSamples; i++ { - d.Add(float64(i)) - } - - require.Equal(t, 0.0, d.Min()) - require.Equal(t, float64(numSamples-1), d.Max()) - for _, q := range testQuantiles { - require.InEpsilon(t, float64(numSamples)*q, d.Quantile(q), 0.01) - } -} - -func TestTDigestWithDecreasingValues(t *testing.T) { - numSamples := 100000 - opts := testTDigestOptions() - d := NewTDigest(opts) - for i := numSamples - 1; i >= 0; i-- { - d.Add(float64(i)) - } - - require.Equal(t, 0.0, d.Min()) - require.Equal(t, float64(numSamples-1), d.Max()) - for _, q := range testQuantiles { - require.InEpsilon(t, float64(numSamples)*q, d.Quantile(q), 0.01) - } -} - -func TestTDigestWithRandomValues(t *testing.T) { - numSamples := 100000 - maxInt64 := int64(math.MaxInt64) - opts := testTDigestOptions() - d := NewTDigest(opts) - min := math.MaxFloat64 - max := -1.0 - - rand.Seed(100) - for i := 0; i < numSamples; i++ { - v := float64(rand.Int63n(maxInt64)) - min = math.Min(min, v) - max = math.Max(max, v) - d.Add(v) - } - - require.Equal(t, min, d.Min()) - require.Equal(t, max, d.Max()) - for _, q := range testQuantiles { - require.InEpsilon(t, float64(maxInt64)*q, d.Quantile(q), 0.01) - } -} - -func TestTDigestMerge(t *testing.T) { - var ( - numSamples = 100000 - numMerges = 10 - maxInt64 = int64(math.MaxInt64) - opts = testTDigestOptions() - digests = make([]TDigest, 0, numMerges) - ) - for i := 0; i < numMerges; i++ { - digests = append(digests, NewTDigest(opts)) - } - - rand.Seed(100) - for i := 0; i < numSamples; i++ { - val := float64(rand.Int63n(maxInt64)) - digests[i%numMerges].Add(val) - } - - merged := NewTDigest(opts) - for i := 0; i < numMerges; i++ { - merged.Merge(digests[i]) - } - - for _, q := range testQuantiles { - require.InEpsilon(t, float64(maxInt64)*q, merged.Quantile(q), 0.01) - } -} - -func TestTDigestClose(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts).(*tDigest) - require.False(t, d.closed) - - // Close the t-digest. - d.Close() - require.True(t, d.closed) - - // Close the t-digest again, should be a no-op. - d.Close() - require.True(t, d.closed) -} - -func TestTDigestAppendCentroid(t *testing.T) { - centroidsPool := NewCentroidsPool( - []pool.Bucket{ - {Capacity: 1, Count: 1}, - {Capacity: 2, Count: 1}, - }, nil) - centroidsPool.Init() - opts := testTDigestOptions().SetCentroidsPool(centroidsPool) - d := NewTDigest(opts).(*tDigest) - - centroids := centroidsPool.Get(1) - require.Equal(t, 1, cap(centroids)) - - inputs := []Centroid{ - {Mean: 1.0, Weight: 0.5}, - {Mean: 2.0, Weight: 0.8}, - } - - // Append one centroid, still under capacity. - centroids = d.appendCentroid(centroids, inputs[0]) - require.Equal(t, []Centroid{inputs[0]}, centroids) - require.Equal(t, 1, cap(centroids)) - - // Append another centroid, which causes the capacity to grow. - centroids = d.appendCentroid(centroids, inputs[1]) - require.Equal(t, inputs, centroids) - require.Equal(t, 2, cap(centroids)) -} - -func TestTDigestCompress(t *testing.T) { - opts := testTDigestOptions() - d := NewTDigest(opts).(*tDigest) - - var result []Centroid - d.mergeCentroidFn = func( - currIndex float64, - currWeight float64, - totalWeight float64, - c Centroid, - mergeResult []Centroid, - ) (float64, []Centroid) { - result = append(result, c) - return currIndex + 1, result - } - d.merged = []Centroid{ - {Mean: 1.0, Weight: 0.5}, - {Mean: 2.0, Weight: 0.8}, - } - d.mergedWeight = 1.3 - d.unmerged = []Centroid{ - {Mean: 5.0, Weight: 1.2}, - {Mean: 4.0, Weight: 2.0}, - {Mean: 6.0, Weight: 1.5}, - } - d.unmergedWeight = 4.7 - - d.compress() - - expected := []Centroid{ - {Mean: 1.0, Weight: 0.5}, - {Mean: 2.0, Weight: 0.8}, - {Mean: 4.0, Weight: 2.0}, - {Mean: 5.0, Weight: 1.2}, - {Mean: 6.0, Weight: 1.5}, - } - require.Equal(t, expected, d.merged) - require.Equal(t, 6.0, d.mergedWeight) - require.Equal(t, 0, len(d.unmerged)) - require.Equal(t, 0.0, d.unmergedWeight) -} diff --git a/src/aggregator/aggregation/quantile/tdigest/types.go b/src/aggregator/aggregation/quantile/tdigest/types.go deleted file mode 100644 index 5fece5d810..0000000000 --- a/src/aggregator/aggregation/quantile/tdigest/types.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tdigest - -// Centroid represents the center of a cluster. -type Centroid struct { - Mean float64 - Weight float64 -} - -// CentroidsPool provides a pool for variable-sized centroid slices. -type CentroidsPool interface { - // Init initializes the pool. - Init() - - // Get provides a centroid slice from the pool. - Get(capacity int) []Centroid - - // Put returns a centroid slice to the pool. - Put(value []Centroid) -} - -// TDigest is the t-digest interface. -type TDigest interface { - - // Merged returns the merged centroids. - Merged() []Centroid - - // Unmerged returns the unmerged centroids. - Unmerged() []Centroid - - // Add adds a value. - Add(value float64) - - // Min returns the minimum value. - Min() float64 - - // Max returns the maximum value. - Max() float64 - - // Quantile returns the quantile value. - Quantile(q float64) float64 - - // Merge merges another t-digest. - Merge(tdigest TDigest) - - // Close clsoes the t-digest. - Close() - - // Reset resets the t-digest. - Reset() -} - -// Options provides a set of t-digest options. -type Options interface { - // SetCompression sets the compression. - SetCompression(value float64) Options - - // Compression returns the compression. - Compression() float64 - - // SetPrecision sets the quantile precision. - SetPrecision(value int) Options - - // Precision returns the quantile precision. - Precision() int - - // SetCentroidsPool sets the centroids pool. - SetCentroidsPool(value CentroidsPool) Options - - // CentroidsPool returns the centroids pool. - CentroidsPool() CentroidsPool - - // Validate validates the options. - Validate() error -} From e159616373684c18dfa76101464238c2376fcc10 Mon Sep 17 00:00:00 2001 From: Matt Way Date: Thu, 29 Oct 2020 21:25:26 -0400 Subject: [PATCH 07/47] Remove old code: aggregator/handler/trafficcontrol (#2821) --- .../handler/trafficcontrol/config.go | 98 ------------- .../handler/trafficcontrol/controller.go | 125 ----------------- .../handler/trafficcontrol/controller_test.go | 131 ------------------ .../handler/trafficcontrol/options.go | 130 ----------------- 4 files changed, 484 deletions(-) delete mode 100644 src/aggregator/aggregator/handler/trafficcontrol/config.go delete mode 100644 src/aggregator/aggregator/handler/trafficcontrol/controller.go delete mode 100644 src/aggregator/aggregator/handler/trafficcontrol/controller_test.go delete mode 100644 src/aggregator/aggregator/handler/trafficcontrol/options.go diff --git a/src/aggregator/aggregator/handler/trafficcontrol/config.go b/src/aggregator/aggregator/handler/trafficcontrol/config.go deleted file mode 100644 index e4e79f45c2..0000000000 --- a/src/aggregator/aggregator/handler/trafficcontrol/config.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package trafficcontrol - -import ( - "fmt" - "time" - - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/x/instrument" -) - -// Type defines the type of the traffic controller. -type Type string - -var ( - // TrafficEnabler enables the traffic when the runtime value equals to true. - TrafficEnabler Type = "trafficEnabler" - // TrafficDisabler disables the traffic when the runtime value equals to true. - TrafficDisabler Type = "trafficDisabler" - - validTypes = []Type{ - TrafficEnabler, - TrafficDisabler, - } -) - -// UnmarshalYAML unmarshals TrafficControllerType from yaml. -func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error { - var str string - if err := unmarshal(&str); err != nil { - return err - } - var validStrings []string - for _, validType := range validTypes { - validString := string(validType) - if validString == str { - *t = validType - return nil - } - validStrings = append(validStrings, validString) - } - - return fmt.Errorf("invalid traffic controller type %s, valid types are: %v", str, validStrings) -} - -// Configuration configures the traffic controller. -type Configuration struct { - Type Type `yaml:"type"` - DefaultValue *bool `yaml:"defaultValue"` - RuntimeKey string `yaml:"runtimeKey" validate:"nonzero"` - InitTimeout *time.Duration `yaml:"initTimeout"` -} - -// NewTrafficController creates a new traffic controller. -func (c *Configuration) NewTrafficController( - store kv.Store, - instrumentOpts instrument.Options, -) (Controller, error) { - opts := NewOptions(). - SetStore(store). - SetRuntimeKey(c.RuntimeKey). - SetInstrumentOptions(instrumentOpts) - if c.DefaultValue != nil { - opts = opts.SetDefaultValue(*c.DefaultValue) - } - if c.InitTimeout != nil { - opts = opts.SetInitTimeout(*c.InitTimeout) - } - var tc Controller - if c.Type == TrafficEnabler { - tc = NewTrafficEnabler(opts) - } else { - tc = NewTrafficDisabler(opts) - } - if err := tc.Init(); err != nil { - return nil, err - } - return tc, nil -} diff --git a/src/aggregator/aggregator/handler/trafficcontrol/controller.go b/src/aggregator/aggregator/handler/trafficcontrol/controller.go deleted file mode 100644 index cb554d27d0..0000000000 --- a/src/aggregator/aggregator/handler/trafficcontrol/controller.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package trafficcontrol - -import ( - "time" - - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/cluster/kv/util" - "github.com/m3db/m3/src/x/watch" - - "go.uber.org/atomic" -) - -const ( - defaultInitTimeout = 2 * time.Second -) - -// Controller controls traffic. -type Controller interface { - // Allow returns true if traffic is allowed. - Allow() bool - - // Init initializes the traffic controller to watch the runtime updates. - Init() error - - // Close closes the traffic controller. - Close() -} - -type trafficEnabler struct { - enabled *atomic.Bool - value watch.Value - opts Options -} - -// NewTrafficEnabler creates a new traffic enabler. -func NewTrafficEnabler(opts Options) Controller { - enabled := atomic.NewBool(opts.DefaultValue()) - iOpts := opts.InstrumentOptions() - newUpdatableFn := func() (watch.Updatable, error) { - w, err := opts.Store().Watch(opts.RuntimeKey()) - return w, err - } - getFn := func(updatable watch.Updatable) (interface{}, error) { - return updatable.(kv.ValueWatch).Get(), nil - } - processFn := func(update interface{}) error { - b, err := util.BoolFromValue( - update.(kv.Value), - opts.RuntimeKey(), - opts.DefaultValue(), - util.NewOptions().SetLogger(iOpts.Logger()), - ) - if err != nil { - return err - } - enabled.Store(b) - return nil - } - vOptions := watch.NewOptions(). - SetInitWatchTimeout(opts.InitTimeout()). - SetInstrumentOptions(iOpts). - SetNewUpdatableFn(newUpdatableFn). - SetGetUpdateFn(getFn). - SetProcessFn(processFn). - SetKey(opts.RuntimeKey()) - return &trafficEnabler{ - enabled: enabled, - value: watch.NewValue(vOptions), - opts: opts, - } -} - -func (c *trafficEnabler) Init() error { - err := c.value.Watch() - if err == nil { - return nil - } - if _, ok := err.(watch.CreateWatchError); ok { - return err - } - return nil -} - -func (c *trafficEnabler) Close() { - c.value.Unwatch() -} - -func (c *trafficEnabler) Allow() bool { - return c.enabled.Load() -} - -type trafficDisabler struct { - Controller -} - -// NewTrafficDisabler creates a new traffic disabler. -func NewTrafficDisabler(opts Options) Controller { - return &trafficDisabler{ - Controller: NewTrafficEnabler(opts), - } -} - -func (c *trafficDisabler) Allow() bool { - return !c.Controller.Allow() -} diff --git a/src/aggregator/aggregator/handler/trafficcontrol/controller_test.go b/src/aggregator/aggregator/handler/trafficcontrol/controller_test.go deleted file mode 100644 index 2970dec845..0000000000 --- a/src/aggregator/aggregator/handler/trafficcontrol/controller_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package trafficcontrol - -import ( - "testing" - "time" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv/mem" - - "github.com/fortytw2/leaktest" - "github.com/stretchr/testify/require" -) - -func TestTrafficControllerWithoutInitialKVValue(t *testing.T) { - defer leaktest.Check(t)() - - store := mem.NewStore() - key := "testKey" - opts := NewOptions(). - SetStore(store). - SetRuntimeKey(key). - SetDefaultValue(true). - SetInitTimeout(200 * time.Millisecond) - enabler := NewTrafficEnabler(opts).(*trafficEnabler) - disabler := NewTrafficDisabler(opts) - require.True(t, enabler.enabled.Load()) - require.True(t, enabler.Allow()) - require.False(t, disabler.Allow()) - - require.NoError(t, enabler.Init()) - defer enabler.Close() - - require.NoError(t, disabler.Init()) - defer disabler.Close() - - _, err := store.Set(key, &commonpb.BoolProto{Value: false}) - require.NoError(t, err) - - for enabler.enabled.Load() { - time.Sleep(100 * time.Millisecond) - } - require.False(t, enabler.Allow()) - - for !disabler.Allow() { - time.Sleep(100 * time.Millisecond) - } - require.True(t, disabler.Allow()) - - _, err = store.Set(key, &commonpb.BoolProto{Value: true}) - require.NoError(t, err) - - for !enabler.enabled.Load() { - time.Sleep(100 * time.Millisecond) - } - require.True(t, enabler.Allow()) - - for disabler.Allow() { - time.Sleep(100 * time.Millisecond) - } - require.False(t, disabler.Allow()) -} - -func TestTrafficControllerWithInitialKVValue(t *testing.T) { - defer leaktest.Check(t)() - - store := mem.NewStore() - key := "testKey" - _, err := store.Set(key, &commonpb.BoolProto{Value: true}) - require.NoError(t, err) - - opts := NewOptions(). - SetStore(store). - SetRuntimeKey(key). - SetDefaultValue(false). - SetInitTimeout(200 * time.Millisecond) - enabler := NewTrafficEnabler(opts).(*trafficEnabler) - require.NoError(t, enabler.Init()) - defer enabler.Close() - - disabler := NewTrafficDisabler(opts) - require.NoError(t, disabler.Init()) - defer disabler.Close() - - require.True(t, enabler.enabled.Load()) - require.True(t, enabler.Allow()) - require.False(t, disabler.Allow()) - - _, err = store.Set(key, &commonpb.BoolProto{Value: false}) - require.NoError(t, err) - - for enabler.enabled.Load() { - time.Sleep(100 * time.Millisecond) - } - require.False(t, enabler.Allow()) - for !disabler.Allow() { - time.Sleep(100 * time.Millisecond) - } - require.True(t, disabler.Allow()) - - _, err = store.Set(key, &commonpb.BoolProto{Value: true}) - require.NoError(t, err) - - for !enabler.enabled.Load() { - time.Sleep(100 * time.Millisecond) - } - require.True(t, enabler.Allow()) - for disabler.Allow() { - time.Sleep(100 * time.Millisecond) - } - require.False(t, disabler.Allow()) -} diff --git a/src/aggregator/aggregator/handler/trafficcontrol/options.go b/src/aggregator/aggregator/handler/trafficcontrol/options.go deleted file mode 100644 index 04a6f89536..0000000000 --- a/src/aggregator/aggregator/handler/trafficcontrol/options.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package trafficcontrol - -import ( - "time" - - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/x/instrument" -) - -// Options configurates the traffic controller. -type Options interface { - // SetStore sets the kv store. - SetStore(store kv.Store) Options - - // Store returns the kv store. - Store() kv.Store - - // SetDefaultValue sets the default value. - SetDefaultValue(value bool) Options - - // DefaultValue returns the default value. - DefaultValue() bool - - // SetRuntimeKey sets the runtime enable key, - // which will override the default enabled value when present. - SetRuntimeKey(value string) Options - - // RuntimeKey returns the runtime enable key, - // which will override the default enabled value when present. - RuntimeKey() string - - // SetInitTimeout sets the init timeout. - SetInitTimeout(value time.Duration) Options - - // InitTimeout returns the init timeout. - InitTimeout() time.Duration - - // SetInstrumentOptions sets the instrument options. - SetInstrumentOptions(value instrument.Options) Options - - // InstrumentOptions returns the instrument options. - InstrumentOptions() instrument.Options -} - -type trafficControlOptions struct { - store kv.Store - defaultValue bool - runtimeKey string - initTimeout time.Duration - instrumentOpts instrument.Options -} - -// NewOptions creates new Options. -func NewOptions() Options { - return &trafficControlOptions{ - initTimeout: defaultInitTimeout, - defaultValue: false, - instrumentOpts: instrument.NewOptions(), - } -} - -func (o *trafficControlOptions) SetStore(store kv.Store) Options { - opts := *o - opts.store = store - return &opts -} - -func (o *trafficControlOptions) Store() kv.Store { - return o.store -} - -func (o *trafficControlOptions) SetDefaultValue(value bool) Options { - opts := *o - opts.defaultValue = value - return &opts -} - -func (o *trafficControlOptions) DefaultValue() bool { - return o.defaultValue -} - -func (o *trafficControlOptions) SetRuntimeKey(value string) Options { - opts := *o - opts.runtimeKey = value - return &opts -} - -func (o *trafficControlOptions) RuntimeKey() string { - return o.runtimeKey -} - -func (o *trafficControlOptions) SetInitTimeout(value time.Duration) Options { - opts := *o - opts.initTimeout = value - return &opts -} - -func (o *trafficControlOptions) InitTimeout() time.Duration { - return o.initTimeout -} - -func (o *trafficControlOptions) SetInstrumentOptions(value instrument.Options) Options { - opts := *o - opts.instrumentOpts = value - return &opts -} - -func (o *trafficControlOptions) InstrumentOptions() instrument.Options { - return o.instrumentOpts -} From 9681fcdd40010b077bfda2ec023a2115fea2e4ba Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Thu, 29 Oct 2020 21:51:28 -0400 Subject: [PATCH 08/47] [proto] Allow zero-alloc reuse of AggregatedMetric protobuf payloads (#2823) --- .../handler/writer/protobuf_test.go | 5 +- .../server/m3msg/protobuf_handler.go | 13 +- src/collector/integration/defaults.go | 8 +- .../encoding/protobuf/aggregated_decoder.go | 13 +- .../aggregated_decoder_benchmark_test.go | 49 +++++++ .../encoding/protobuf/aggregated_encoder.go | 2 +- .../protobuf/aggregated_roundtrip_test.go | 12 +- src/metrics/encoding/protobuf/reset.go | 18 +-- src/metrics/encoding/protobuf/reset_test.go | 8 +- .../protobuf/unaggregated_encoder_test.go | 44 +++--- .../generated/proto/policypb/policy.pb.go | 117 +++++++-------- .../generated/proto/policypb/policy.proto | 5 +- .../match_rule_update_stress_test.go | 8 +- src/metrics/metadata/metadata_test.go | 40 +++--- src/metrics/metric/aggregated/types_test.go | 8 +- src/metrics/metric/unaggregated/types_test.go | 12 +- src/metrics/policy/policy_test.go | 28 ++-- src/metrics/policy/resolution.go | 7 +- src/metrics/policy/retention.go | 7 +- src/metrics/policy/storage_policy.go | 10 +- src/metrics/policy/storage_policy_test.go | 20 +-- src/metrics/rules/convert_test.go | 40 +++--- src/metrics/rules/mapping_test.go | 32 ++--- src/metrics/rules/rollup_target_test.go | 56 ++++---- src/metrics/rules/rollup_test.go | 36 ++--- src/metrics/rules/ruleset_test.go | 136 +++++++++--------- src/metrics/rules/store/kv/store_test.go | 44 +++--- src/query/remote/codecs_test.go | 20 +-- 28 files changed, 412 insertions(+), 386 deletions(-) create mode 100644 src/metrics/encoding/protobuf/aggregated_decoder_benchmark_test.go diff --git a/src/aggregator/aggregator/handler/writer/protobuf_test.go b/src/aggregator/aggregator/handler/writer/protobuf_test.go index 727be98b27..8fe686d8d2 100644 --- a/src/aggregator/aggregator/handler/writer/protobuf_test.go +++ b/src/aggregator/aggregator/handler/writer/protobuf_test.go @@ -121,10 +121,9 @@ func TestProtobufWriterWrite(t *testing.T) { actualData := make(map[uint32][]decodeData) writer.p.(*producer.MockProducer).EXPECT().Produce(gomock.Any()).Do(func(m producer.Message) error { d := protobuf.NewAggregatedDecoder(nil) - d.Decode(m.Bytes()) + require.NoError(t, d.Decode(m.Bytes())) s := m.Shard() - sp, err := d.StoragePolicy() - require.NoError(t, err) + sp := d.StoragePolicy() actualData[s] = append(actualData[s], decodeData{ MetricWithStoragePolicy: aggregated.MetricWithStoragePolicy{ Metric: aggregated.Metric{ diff --git a/src/cmd/services/m3coordinator/server/m3msg/protobuf_handler.go b/src/cmd/services/m3coordinator/server/m3msg/protobuf_handler.go index 7589b95595..ffca0e0c1d 100644 --- a/src/cmd/services/m3coordinator/server/m3msg/protobuf_handler.go +++ b/src/cmd/services/m3coordinator/server/m3msg/protobuf_handler.go @@ -47,7 +47,6 @@ type handlerMetrics struct { metricAccepted tally.Counter droppedMetricBlackholePolicy tally.Counter droppedMetricDecodeError tally.Counter - droppedMetricDecodeMalformed tally.Counter } func newHandlerMetrics(scope tally.Scope) handlerMetrics { @@ -61,9 +60,6 @@ func newHandlerMetrics(scope tally.Scope) handlerMetrics { droppedMetricBlackholePolicy: messageScope.Tagged(map[string]string{ "reason": "blackhole-policy", }).Counter("dropped"), - droppedMetricDecodeMalformed: messageScope.Tagged(map[string]string{ - "reason": "decode-malformed", - }).Counter("dropped"), } } @@ -111,18 +107,11 @@ func (h *pbHandler) Process(msg consumer.Message) { h.m.droppedMetricDecodeError.Inc(1) return } - sp, err := dec.StoragePolicy() - if err != nil { - h.logger.Error("invalid storage policy", zap.Error(err)) - h.m.droppedMetricDecodeMalformed.Inc(1) - return - } - h.m.metricAccepted.Inc(1) h.wg.Add(1) r := NewProtobufCallback(msg, dec, h.wg) - + sp := dec.StoragePolicy() // If storage policy is blackholed, ack the message immediately and don't // bother passing down the write path. for _, blackholeSp := range h.blackholePolicies { diff --git a/src/collector/integration/defaults.go b/src/collector/integration/defaults.go index 284c045cce..5a2d7c712a 100644 --- a/src/collector/integration/defaults.go +++ b/src/collector/integration/defaults.go @@ -101,11 +101,11 @@ func defaultMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -142,11 +142,11 @@ func defaultRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, diff --git a/src/metrics/encoding/protobuf/aggregated_decoder.go b/src/metrics/encoding/protobuf/aggregated_decoder.go index c90fdc0a85..ca9052c44c 100644 --- a/src/metrics/encoding/protobuf/aggregated_decoder.go +++ b/src/metrics/encoding/protobuf/aggregated_decoder.go @@ -29,6 +29,7 @@ import ( type AggregatedDecoder struct { pool AggregatedDecoderPool pb metricpb.AggregatedMetric + sp policy.StoragePolicy } // NewAggregatedDecoder creates an aggregated decoder. @@ -40,7 +41,10 @@ func NewAggregatedDecoder(p AggregatedDecoderPool) *AggregatedDecoder { // Decode decodes the aggregated metric from the given bytes. func (d *AggregatedDecoder) Decode(b []byte) error { - return d.pb.Unmarshal(b) + if err := d.pb.Unmarshal(b); err != nil { + return err + } + return d.sp.FromProto(d.pb.Metric.StoragePolicy) } // ID returns the decoded id. @@ -59,8 +63,8 @@ func (d AggregatedDecoder) Value() float64 { } // StoragePolicy returns the decoded storage policy. -func (d AggregatedDecoder) StoragePolicy() (policy.StoragePolicy, error) { - return policy.NewStoragePolicyFromProto(&d.pb.Metric.StoragePolicy) +func (d AggregatedDecoder) StoragePolicy() policy.StoragePolicy { + return d.sp } // EncodeNanos returns the decoded encodeNanos. @@ -70,7 +74,8 @@ func (d AggregatedDecoder) EncodeNanos() int64 { // Close closes the decoder. func (d *AggregatedDecoder) Close() { - resetAggregatedMetricProto(&d.pb) + d.sp = policy.StoragePolicy{} + ReuseAggregatedMetricProto(&d.pb) if d.pool != nil { d.pool.Put(d) } diff --git a/src/metrics/encoding/protobuf/aggregated_decoder_benchmark_test.go b/src/metrics/encoding/protobuf/aggregated_decoder_benchmark_test.go new file mode 100644 index 0000000000..56e83b4891 --- /dev/null +++ b/src/metrics/encoding/protobuf/aggregated_decoder_benchmark_test.go @@ -0,0 +1,49 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package protobuf + +import ( + "runtime" + "testing" + + "github.com/m3db/m3/src/metrics/policy" +) + +func BenchmarkDecodeStoragePolicy(b *testing.B) { + var ( + enc = NewAggregatedEncoder(nil) + dec = NewAggregatedDecoder(nil) + sp policy.StoragePolicy + ) + if err := enc.Encode(testAggregatedMetric1, 2000); err != nil { + b.Fatal(err) + } + + buf := enc.Buffer().Bytes() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = dec.Decode(buf) + sp = dec.StoragePolicy() + dec.Close() + } + runtime.KeepAlive(sp) +} diff --git a/src/metrics/encoding/protobuf/aggregated_encoder.go b/src/metrics/encoding/protobuf/aggregated_encoder.go index bdd3e1930f..c27c60226b 100644 --- a/src/metrics/encoding/protobuf/aggregated_encoder.go +++ b/src/metrics/encoding/protobuf/aggregated_encoder.go @@ -55,7 +55,7 @@ func (enc *aggregatedEncoder) Encode( m aggregated.MetricWithStoragePolicy, encodedAtNanos int64, ) error { - resetAggregatedMetricProto(&enc.pb) + ReuseAggregatedMetricProto(&enc.pb) if err := m.ToProto(&enc.pb.Metric); err != nil { return err } diff --git a/src/metrics/encoding/protobuf/aggregated_roundtrip_test.go b/src/metrics/encoding/protobuf/aggregated_roundtrip_test.go index f30544e32a..516d41f6e6 100644 --- a/src/metrics/encoding/protobuf/aggregated_roundtrip_test.go +++ b/src/metrics/encoding/protobuf/aggregated_roundtrip_test.go @@ -57,8 +57,7 @@ func TestAggregatedEncoderDecoder_RoundTrip(t *testing.T) { require.NoError(t, enc.Encode(testAggregatedMetric1, 2000)) require.NoError(t, dec.Decode(enc.Buffer().Bytes())) require.Equal(t, int64(2000), dec.EncodeNanos()) - sp, err := dec.StoragePolicy() - require.NoError(t, err) + sp := dec.StoragePolicy() require.Equal(t, testAggregatedMetric1.StoragePolicy, sp) require.Equal(t, string(testAggregatedMetric1.ID), string(dec.ID())) require.Equal(t, testAggregatedMetric1.TimeNanos, dec.TimeNanos()) @@ -77,8 +76,7 @@ func TestAggregatedEncoderDecoder_WithBytesPool(t *testing.T) { require.NoError(t, enc.Encode(testAggregatedMetric1, 2000)) require.NoError(t, dec.Decode(enc.Buffer().Bytes())) require.Equal(t, int64(2000), dec.EncodeNanos()) - sp, err := dec.StoragePolicy() - require.NoError(t, err) + sp := dec.StoragePolicy() require.Equal(t, testAggregatedMetric1.StoragePolicy, sp) require.Equal(t, string(testAggregatedMetric1.ID), string(dec.ID())) require.Equal(t, testAggregatedMetric1.TimeNanos, dec.TimeNanos()) @@ -91,8 +89,7 @@ func TestAggregatedEncoderDecoder_ResetProtobuf(t *testing.T) { require.NoError(t, enc.Encode(testAggregatedMetric1, 2000)) require.NoError(t, dec.Decode(enc.Buffer().Bytes())) require.Equal(t, int64(2000), dec.EncodeNanos()) - sp, err := dec.StoragePolicy() - require.NoError(t, err) + sp := dec.StoragePolicy() require.Equal(t, testAggregatedMetric1.StoragePolicy, sp) require.Equal(t, string(testAggregatedMetric1.ID), string(dec.ID())) require.Equal(t, testAggregatedMetric1.TimeNanos, dec.TimeNanos()) @@ -105,8 +102,7 @@ func TestAggregatedEncoderDecoder_ResetProtobuf(t *testing.T) { require.NoError(t, enc.Encode(testAggregatedMetric2, 3000)) require.NoError(t, dec.Decode(enc.Buffer().Bytes())) require.Equal(t, int64(3000), dec.EncodeNanos()) - sp, err = dec.StoragePolicy() - require.NoError(t, err) + sp = dec.StoragePolicy() require.Equal(t, testAggregatedMetric2.StoragePolicy, sp) require.Equal(t, string(testAggregatedMetric2.ID), string(dec.ID())) require.Equal(t, testAggregatedMetric2.TimeNanos, dec.TimeNanos()) diff --git a/src/metrics/encoding/protobuf/reset.go b/src/metrics/encoding/protobuf/reset.go index 6a97fdff5e..077e17c48f 100644 --- a/src/metrics/encoding/protobuf/reset.go +++ b/src/metrics/encoding/protobuf/reset.go @@ -24,14 +24,6 @@ import ( "github.com/m3db/m3/src/metrics/generated/proto/metricpb" ) -func resetAggregatedMetricProto(pb *metricpb.AggregatedMetric) { - if pb == nil { - return - } - resetTimedMetricWithStoragePolicyProto(&pb.Metric) - pb.EncodeNanos = 0 -} - // ReuseMetricWithMetadatasProto allows for zero-alloc reuse of // *metricpb.MetricWithMetadatas by deep resetting the internal slices // and when using gogoprotobuf's unmarshal function will reuse the slices @@ -60,6 +52,16 @@ func ReuseMetricWithMetadatasProto(pb *metricpb.MetricWithMetadatas) { resetTimedMetricWithStoragePolicyProto(pb.TimedMetricWithStoragePolicy) } +// ReuseAggregatedMetricProto allows for zero-alloc reuse of +// *metricpb.AggregatedMetric +func ReuseAggregatedMetricProto(pb *metricpb.AggregatedMetric) { + if pb == nil { + return + } + resetTimedMetricWithStoragePolicyProto(&pb.Metric) + pb.EncodeNanos = 0 +} + func resetCounterWithMetadatasProto(pb *metricpb.CounterWithMetadatas) { if pb == nil { return diff --git a/src/metrics/encoding/protobuf/reset_test.go b/src/metrics/encoding/protobuf/reset_test.go index a3a8b3eb94..e489862669 100644 --- a/src/metrics/encoding/protobuf/reset_test.go +++ b/src/metrics/encoding/protobuf/reset_test.go @@ -118,23 +118,23 @@ func TestReuseMetricWithMetadatasProtoNilProto(t *testing.T) { require.NotPanics(t, func() { ReuseMetricWithMetadatasProto(nil) }) } -func TestResetAggregatedMetricProto(t *testing.T) { +func TestReuseAggregatedMetricProto(t *testing.T) { input := &metricpb.AggregatedMetric{ Metric: metricpb.TimedMetricWithStoragePolicy{ TimedMetric: testTimedMetricBeforeResetProto, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, }, EncodeNanos: 1234, } - resetAggregatedMetricProto(input) + ReuseAggregatedMetricProto(input) require.Equal(t, metricpb.AggregatedMetric{ Metric: metricpb.TimedMetricWithStoragePolicy{ TimedMetric: metricpb.TimedMetric{Id: []byte{}}, diff --git a/src/metrics/encoding/protobuf/unaggregated_encoder_test.go b/src/metrics/encoding/protobuf/unaggregated_encoder_test.go index 57364bd604..7c3ad0026d 100644 --- a/src/metrics/encoding/protobuf/unaggregated_encoder_test.go +++ b/src/metrics/encoding/protobuf/unaggregated_encoder_test.go @@ -324,11 +324,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -338,11 +338,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -379,11 +379,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Count)[0]}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -393,20 +393,20 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (30 * 24 * time.Hour).Nanoseconds(), }, }, @@ -465,11 +465,11 @@ var ( testForwardMetadata1Proto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, }, @@ -490,11 +490,11 @@ var ( testForwardMetadata2Proto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, @@ -521,11 +521,11 @@ var ( testTimedMetadata1Proto = metricpb.TimedMetadata{ AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, }, @@ -533,30 +533,30 @@ var ( testTimedMetadata2Proto = metricpb.TimedMetadata{ AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, } testPassthroughMetadata1Proto = policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, } testPassthroughMetadata2Proto = policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, } diff --git a/src/metrics/generated/proto/policypb/policy.pb.go b/src/metrics/generated/proto/policypb/policy.pb.go index eca5808305..f890d31ca6 100644 --- a/src/metrics/generated/proto/policypb/policy.pb.go +++ b/src/metrics/generated/proto/policypb/policy.pb.go @@ -38,6 +38,7 @@ package policypb import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" +import _ "github.com/gogo/protobuf/gogoproto" import aggregationpb "github.com/m3db/m3/src/metrics/generated/proto/aggregationpb" import io "io" @@ -118,8 +119,8 @@ func (m *Retention) GetPeriod() int64 { } type StoragePolicy struct { - Resolution *Resolution `protobuf:"bytes,1,opt,name=resolution" json:"resolution,omitempty"` - Retention *Retention `protobuf:"bytes,2,opt,name=retention" json:"retention,omitempty"` + Resolution Resolution `protobuf:"bytes,1,opt,name=resolution" json:"resolution"` + Retention Retention `protobuf:"bytes,2,opt,name=retention" json:"retention"` } func (m *StoragePolicy) Reset() { *m = StoragePolicy{} } @@ -127,18 +128,18 @@ func (m *StoragePolicy) String() string { return proto.CompactTextStr func (*StoragePolicy) ProtoMessage() {} func (*StoragePolicy) Descriptor() ([]byte, []int) { return fileDescriptorPolicy, []int{2} } -func (m *StoragePolicy) GetResolution() *Resolution { +func (m *StoragePolicy) GetResolution() Resolution { if m != nil { return m.Resolution } - return nil + return Resolution{} } -func (m *StoragePolicy) GetRetention() *Retention { +func (m *StoragePolicy) GetRetention() Retention { if m != nil { return m.Retention } - return nil + return Retention{} } type Policy struct { @@ -238,26 +239,22 @@ func (m *StoragePolicy) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Resolution != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintPolicy(dAtA, i, uint64(m.Resolution.Size())) - n1, err := m.Resolution.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n1 + dAtA[i] = 0xa + i++ + i = encodeVarintPolicy(dAtA, i, uint64(m.Resolution.Size())) + n1, err := m.Resolution.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - if m.Retention != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPolicy(dAtA, i, uint64(m.Retention.Size())) - n2, err := m.Retention.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n2 + i += n1 + dAtA[i] = 0x12 + i++ + i = encodeVarintPolicy(dAtA, i, uint64(m.Retention.Size())) + n2, err := m.Retention.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n2 return i, nil } @@ -339,14 +336,10 @@ func (m *Retention) Size() (n int) { func (m *StoragePolicy) Size() (n int) { var l int _ = l - if m.Resolution != nil { - l = m.Resolution.Size() - n += 1 + l + sovPolicy(uint64(l)) - } - if m.Retention != nil { - l = m.Retention.Size() - n += 1 + l + sovPolicy(uint64(l)) - } + l = m.Resolution.Size() + n += 1 + l + sovPolicy(uint64(l)) + l = m.Retention.Size() + n += 1 + l + sovPolicy(uint64(l)) return n } @@ -592,9 +585,6 @@ func (m *StoragePolicy) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Resolution == nil { - m.Resolution = &Resolution{} - } if err := m.Resolution.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -625,9 +615,6 @@ func (m *StoragePolicy) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Retention == nil { - m.Retention = &Retention{} - } if err := m.Retention.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -908,30 +895,32 @@ func init() { } var fileDescriptorPolicy = []byte{ - // 390 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0xcd, 0xce, 0xd2, 0x40, - 0x14, 0xfd, 0xca, 0x67, 0xc8, 0xd7, 0x4b, 0x20, 0x75, 0x34, 0x48, 0x8c, 0xa9, 0xa4, 0x6e, 0x88, - 0x8b, 0x36, 0x82, 0x89, 0x2b, 0x4d, 0x50, 0x34, 0x12, 0xa4, 0x90, 0x01, 0x17, 0xba, 0x69, 0xfa, - 0x33, 0xa9, 0x93, 0xd0, 0xce, 0x64, 0x66, 0x08, 0xc2, 0x53, 0xb8, 0xf1, 0x9d, 0x5c, 0xfa, 0x08, - 0x06, 0x5f, 0xc4, 0x38, 0x2d, 0xb6, 0x5d, 0xfa, 0xed, 0xe6, 0x9c, 0x7b, 0xef, 0x39, 0xe7, 0xde, - 0x0c, 0xcc, 0x52, 0xaa, 0xbe, 0xec, 0x23, 0x37, 0x66, 0x99, 0x97, 0x4d, 0x92, 0xc8, 0xcb, 0x26, - 0x9e, 0x14, 0xb1, 0x97, 0x11, 0x25, 0x68, 0x2c, 0xbd, 0x94, 0xe4, 0x44, 0x84, 0x8a, 0x24, 0x1e, - 0x17, 0x4c, 0x31, 0x8f, 0xb3, 0x1d, 0x8d, 0x8f, 0x3c, 0x2a, 0x1f, 0xae, 0x66, 0xd1, 0xcd, 0x85, - 0x7e, 0xe8, 0xff, 0xa7, 0x5e, 0x98, 0xa6, 0x82, 0xa4, 0xa1, 0xa2, 0x2c, 0xe7, 0x51, 0x1d, 0x15, - 0xca, 0xce, 0x02, 0x00, 0x13, 0xc9, 0x76, 0xfb, 0xbf, 0x1c, 0x7a, 0x0c, 0x9d, 0x03, 0xcd, 0x13, - 0x76, 0x08, 0x24, 0x3d, 0x91, 0x81, 0x31, 0x34, 0x46, 0xd7, 0x18, 0x0a, 0x6a, 0x43, 0x4f, 0x04, - 0x3d, 0x02, 0x93, 0x0b, 0x12, 0x53, 0x49, 0x59, 0x3e, 0x68, 0xe9, 0x72, 0x45, 0x38, 0x4f, 0xc0, - 0xc4, 0x44, 0x91, 0x5c, 0x6b, 0xf5, 0xa1, 0xcd, 0x89, 0xa0, 0x2c, 0x29, 0x65, 0x4a, 0xe4, 0x7c, - 0x85, 0xee, 0x46, 0x31, 0x11, 0xa6, 0x64, 0xad, 0x97, 0x42, 0xcf, 0x01, 0xc4, 0xbf, 0x08, 0xba, - 0xb9, 0x33, 0xbe, 0xef, 0x5e, 0x36, 0x76, 0xab, 0x78, 0xb8, 0xd6, 0x87, 0x9e, 0x81, 0x29, 0x2e, - 0x5e, 0x3a, 0x49, 0x67, 0x7c, 0xaf, 0x3e, 0x54, 0x96, 0x70, 0xd5, 0xe5, 0x7c, 0x37, 0xa0, 0x5d, - 0x7a, 0xbe, 0x82, 0x9e, 0x2c, 0x42, 0x04, 0xc5, 0x4c, 0xe9, 0xfb, 0xa0, 0x92, 0x68, 0x84, 0xc4, - 0x5d, 0xd9, 0xc8, 0xbc, 0x80, 0xbb, 0xb5, 0x5b, 0x06, 0xea, 0xc8, 0x89, 0x1c, 0xb4, 0x86, 0xd7, - 0xa3, 0xde, 0xd8, 0x76, 0x1b, 0x37, 0x77, 0xa7, 0x15, 0xda, 0x1e, 0x39, 0xc1, 0x56, 0xd8, 0x24, - 0xe4, 0xd3, 0x97, 0x00, 0x33, 0xc1, 0x78, 0x29, 0x7d, 0x03, 0x77, 0xfc, 0x95, 0xff, 0xd6, 0xba, - 0x42, 0x5d, 0x30, 0x67, 0x78, 0xb5, 0x0e, 0x96, 0x1f, 0x37, 0x5b, 0xcb, 0x40, 0x7d, 0x40, 0x1a, - 0xce, 0xdf, 0x05, 0x2b, 0xff, 0xc3, 0xa7, 0x60, 0x39, 0xdd, 0xbe, 0x79, 0x6f, 0xb5, 0x5e, 0xcf, - 0x7f, 0x9c, 0x6d, 0xe3, 0xe7, 0xd9, 0x36, 0x7e, 0x9d, 0x6d, 0xe3, 0xdb, 0x6f, 0xfb, 0xea, 0xf3, - 0x8b, 0x5b, 0x7e, 0xba, 0xa8, 0xad, 0xf1, 0xe4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x58, - 0xd9, 0xc4, 0xb6, 0x02, 0x00, 0x00, + // 420 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0x5d, 0x6b, 0x13, 0x41, + 0x14, 0xcd, 0xa4, 0x25, 0x34, 0x37, 0xa4, 0xac, 0xa3, 0xd4, 0x50, 0x64, 0x5b, 0xd6, 0x97, 0x22, + 0xb8, 0x0b, 0xc9, 0x43, 0x41, 0x50, 0x68, 0x8d, 0x62, 0xa9, 0xdd, 0x94, 0x4d, 0x7c, 0xd0, 0x97, + 0x65, 0x3f, 0xc6, 0x71, 0xa0, 0xbb, 0x33, 0xcc, 0x4c, 0x28, 0xe9, 0xb3, 0x3f, 0xc0, 0x17, 0xff, + 0x53, 0x1f, 0xfd, 0x05, 0x22, 0xf1, 0x8f, 0x48, 0x66, 0x27, 0xee, 0xee, 0xa3, 0xbe, 0xdd, 0x73, + 0xee, 0x3d, 0xe7, 0x1e, 0xee, 0x0c, 0x4c, 0x29, 0xd3, 0x5f, 0x96, 0xa9, 0x9f, 0xf1, 0x22, 0x28, + 0x26, 0x79, 0x1a, 0x14, 0x93, 0x40, 0xc9, 0x2c, 0x28, 0x88, 0x96, 0x2c, 0x53, 0x01, 0x25, 0x25, + 0x91, 0x89, 0x26, 0x79, 0x20, 0x24, 0xd7, 0x3c, 0x10, 0xfc, 0x86, 0x65, 0x2b, 0x91, 0xda, 0xc2, + 0x37, 0x2c, 0xde, 0xdb, 0xd2, 0x87, 0xcf, 0x1b, 0x7e, 0x94, 0x53, 0x5e, 0xc9, 0xd2, 0xe5, 0x67, + 0x83, 0x2a, 0x8f, 0x4d, 0x55, 0x09, 0x0f, 0xc3, 0x7f, 0x5c, 0x9f, 0x50, 0x2a, 0x09, 0x4d, 0x34, + 0xe3, 0xa5, 0x48, 0x9b, 0xa8, 0xf2, 0xf3, 0x2e, 0x01, 0x22, 0xa2, 0xf8, 0xcd, 0x72, 0xc3, 0xe1, + 0x23, 0x18, 0xdc, 0xb2, 0x32, 0xe7, 0xb7, 0xb1, 0x62, 0x77, 0x64, 0x84, 0x8e, 0xd1, 0xc9, 0x4e, + 0x04, 0x15, 0x35, 0x67, 0x77, 0x04, 0x3f, 0x81, 0xbe, 0x90, 0x24, 0x63, 0x8a, 0xf1, 0x72, 0xd4, + 0x35, 0xed, 0x9a, 0xf0, 0x9e, 0x42, 0x3f, 0x22, 0x9a, 0x94, 0xc6, 0xeb, 0x00, 0x7a, 0x82, 0x48, + 0xc6, 0x73, 0x6b, 0x63, 0x91, 0xf7, 0x15, 0xc1, 0x70, 0xae, 0xb9, 0x4c, 0x28, 0xb9, 0x36, 0x47, + 0xc0, 0x2f, 0x00, 0xe4, 0xdf, 0x0c, 0x66, 0x7a, 0x30, 0x7e, 0xe4, 0x6f, 0x2f, 0xe4, 0xd7, 0xf9, + 0xce, 0x77, 0xef, 0x7f, 0x1e, 0x75, 0xa2, 0xc6, 0x34, 0x3e, 0x85, 0xbe, 0xdc, 0xae, 0x34, 0x81, + 0x06, 0xe3, 0x87, 0x4d, 0xa9, 0x6d, 0x59, 0x65, 0x3d, 0xeb, 0x7d, 0x47, 0xd0, 0xb3, 0xfb, 0x5f, + 0xc1, 0xbe, 0xaa, 0x02, 0xc5, 0x95, 0xd2, 0x66, 0x78, 0x5c, 0x1b, 0xb5, 0x02, 0x47, 0x43, 0xd5, + 0xca, 0x7f, 0x09, 0x0f, 0x1a, 0x87, 0x8d, 0xf5, 0x4a, 0x10, 0x35, 0xea, 0x1e, 0xef, 0x9c, 0xec, + 0x8f, 0x5d, 0xbf, 0xf5, 0x00, 0xfe, 0x59, 0x8d, 0x16, 0x2b, 0x41, 0x22, 0x27, 0x69, 0x13, 0xea, + 0xd9, 0x4b, 0x80, 0xa9, 0xe4, 0xc2, 0x5a, 0xef, 0xc1, 0x6e, 0x38, 0x0b, 0xdf, 0x38, 0x1d, 0x3c, + 0x84, 0xfe, 0x34, 0x9a, 0x5d, 0xc7, 0x57, 0x1f, 0xe6, 0x0b, 0x07, 0xe1, 0x03, 0xc0, 0x06, 0x5e, + 0xbc, 0x8d, 0x67, 0xe1, 0xfb, 0x8f, 0xf1, 0xd5, 0xd9, 0xe2, 0xf5, 0x3b, 0xa7, 0x7b, 0x7e, 0x71, + 0xbf, 0x76, 0xd1, 0x8f, 0xb5, 0x8b, 0x7e, 0xad, 0x5d, 0xf4, 0xed, 0xb7, 0xdb, 0xf9, 0x74, 0xfa, + 0x9f, 0x1f, 0x36, 0xed, 0x19, 0x3c, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff, 0x16, 0x63, 0x9d, 0x5c, + 0xf2, 0x02, 0x00, 0x00, } diff --git a/src/metrics/generated/proto/policypb/policy.proto b/src/metrics/generated/proto/policypb/policy.proto index a3e183a001..0d6ca47fdc 100644 --- a/src/metrics/generated/proto/policypb/policy.proto +++ b/src/metrics/generated/proto/policypb/policy.proto @@ -24,6 +24,7 @@ option go_package = "github.com/m3db/m3/src/metrics/generated/proto/policypb"; package policypb; +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "github.com/m3db/m3/src/metrics/generated/proto/aggregationpb/aggregation.proto"; message Resolution { @@ -36,8 +37,8 @@ message Retention { } message StoragePolicy { - Resolution resolution = 1; - Retention retention = 2; + Resolution resolution = 1 [(gogoproto.nullable) = false]; + Retention retention = 2 [(gogoproto.nullable) = false]; } message Policy { diff --git a/src/metrics/integration/match_rule_update_stress_test.go b/src/metrics/integration/match_rule_update_stress_test.go index fb4603bd7e..7fda3c0938 100644 --- a/src/metrics/integration/match_rule_update_stress_test.go +++ b/src/metrics/integration/match_rule_update_stress_test.go @@ -327,11 +327,11 @@ func stressTestMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -368,11 +368,11 @@ func stressTestRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, diff --git a/src/metrics/metadata/metadata_test.go b/src/metrics/metadata/metadata_test.go index abd8087061..d1d834b8d0 100644 --- a/src/metrics/metadata/metadata_test.go +++ b/src/metrics/metadata/metadata_test.go @@ -239,11 +239,11 @@ var ( testSmallForwardMetadataProto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, }, @@ -264,11 +264,11 @@ var ( testLargeForwardMetadataProto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, @@ -297,11 +297,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, @@ -321,20 +321,20 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, }, { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (30 * 24 * time.Hour).Nanoseconds(), }, }, @@ -373,11 +373,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -387,11 +387,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -428,11 +428,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Count)[0]}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -442,20 +442,20 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (30 * 24 * time.Hour).Nanoseconds(), }, }, diff --git a/src/metrics/metric/aggregated/types_test.go b/src/metrics/metric/aggregated/types_test.go index 5448d730f9..9d27d8de02 100644 --- a/src/metrics/metric/aggregated/types_test.go +++ b/src/metrics/metric/aggregated/types_test.go @@ -142,11 +142,11 @@ var ( testForwardMetadata1Proto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (12 * time.Hour).Nanoseconds(), }, }, @@ -167,11 +167,11 @@ var ( testForwardMetadata2Proto = metricpb.ForwardMetadata{ AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Sum)[0]}, StoragePolicy: policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, diff --git a/src/metrics/metric/unaggregated/types_test.go b/src/metrics/metric/unaggregated/types_test.go index 228f22a662..bc770be87f 100644 --- a/src/metrics/metric/unaggregated/types_test.go +++ b/src/metrics/metric/unaggregated/types_test.go @@ -190,11 +190,11 @@ var ( AggregationId: aggregationpb.AggregationID{Id: aggregation.MustCompressTypes(aggregation.Count)[0]}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -204,20 +204,20 @@ var ( AggregationId: aggregationpb.AggregationID{Id: 0}, StoragePolicies: []policypb.StoragePolicy{ { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (6 * time.Hour).Nanoseconds(), }, }, { - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: (30 * 24 * time.Hour).Nanoseconds(), }, }, diff --git a/src/metrics/policy/policy_test.go b/src/metrics/policy/policy_test.go index 2ddd3d77ea..11709ba9b7 100644 --- a/src/metrics/policy/policy_test.go +++ b/src/metrics/policy/policy_test.go @@ -135,11 +135,11 @@ func TestNewPoliciesFromProto(t *testing.T) { input := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -150,11 +150,11 @@ func TestNewPoliciesFromProto(t *testing.T) { }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(240 * time.Hour), }, }, @@ -182,11 +182,11 @@ func TestParsePolicyIntoProto(t *testing.T) { str: "1s:1h", expected: &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -196,11 +196,11 @@ func TestParsePolicyIntoProto(t *testing.T) { str: "1s:1h|Mean", expected: &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, @@ -211,11 +211,11 @@ func TestParsePolicyIntoProto(t *testing.T) { str: "60s:24h|Mean,Count", expected: &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -226,11 +226,11 @@ func TestParsePolicyIntoProto(t *testing.T) { str: "1m:1d|Count,Mean", expected: &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -241,11 +241,11 @@ func TestParsePolicyIntoProto(t *testing.T) { str: "1m@1s:1h|P999,P9999", expected: &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, }, diff --git a/src/metrics/policy/resolution.go b/src/metrics/policy/resolution.go index c2cf1cb829..290ff8e36c 100644 --- a/src/metrics/policy/resolution.go +++ b/src/metrics/policy/resolution.go @@ -36,7 +36,8 @@ const ( var ( emptyResolution Resolution - errNilResolutionProto = errors.New("nil resolution proto message") + emptyResolutionProto = policypb.Resolution{} + errNilResolutionProto = errors.New("empty resolution proto message") ) // Resolution is the sampling resolution for datapoints. @@ -60,8 +61,8 @@ func (r Resolution) ToProto(pb *policypb.Resolution) error { } // FromProto converts the protobuf message to a resolution in place. -func (r *Resolution) FromProto(pb *policypb.Resolution) error { - if pb == nil { +func (r *Resolution) FromProto(pb policypb.Resolution) error { + if pb == emptyResolutionProto { return errNilResolutionProto } precision, err := xtime.UnitFromDuration(time.Duration(pb.Precision)) diff --git a/src/metrics/policy/retention.go b/src/metrics/policy/retention.go index 064b3abd22..cde4b23ae9 100644 --- a/src/metrics/policy/retention.go +++ b/src/metrics/policy/retention.go @@ -30,7 +30,8 @@ import ( ) var ( - errNilRetentionProto = errors.New("nil retention proto message") + errNilRetentionProto = errors.New("empty retention proto message") + emptyRetentionProto = policypb.Retention{} ) // Retention is the retention period for datapoints. @@ -52,8 +53,8 @@ func (r Retention) ToProto(pb *policypb.Retention) { } // FromProto converts the protobuf message to a retention in place. -func (r *Retention) FromProto(pb *policypb.Retention) error { - if pb == nil { +func (r *Retention) FromProto(pb policypb.Retention) error { + if pb == emptyRetentionProto { return errNilRetentionProto } *r = Retention(pb.Period) diff --git a/src/metrics/policy/storage_policy.go b/src/metrics/policy/storage_policy.go index 1c4221a22e..3d8e2905c0 100644 --- a/src/metrics/policy/storage_policy.go +++ b/src/metrics/policy/storage_policy.go @@ -107,16 +107,10 @@ func (p StoragePolicy) Proto() (*policypb.StoragePolicy, error) { // ToProto converts the storage policy to a protobuf message in place. func (p StoragePolicy) ToProto(pb *policypb.StoragePolicy) error { - if pb.Resolution == nil { - pb.Resolution = &policypb.Resolution{} - } - if err := p.resolution.ToProto(pb.Resolution); err != nil { + if err := p.resolution.ToProto(&pb.Resolution); err != nil { return err } - if pb.Retention == nil { - pb.Retention = &policypb.Retention{} - } - p.retention.ToProto(pb.Retention) + p.retention.ToProto(&pb.Retention) return nil } diff --git a/src/metrics/policy/storage_policy_test.go b/src/metrics/policy/storage_policy_test.go index 3390d8a604..f00f67928d 100644 --- a/src/metrics/policy/storage_policy_test.go +++ b/src/metrics/policy/storage_policy_test.go @@ -38,31 +38,31 @@ var ( testStoragePolicy = NewStoragePolicy(10*time.Second, xtime.Second, time.Hour) testBadStoragePolicy = NewStoragePolicy(10*time.Second, xtime.Unit(100), time.Hour) testStoragePolicyProto = policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: (10 * time.Second).Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, } testStoragePolicyProtoNoResolution = policypb.StoragePolicy{ - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, } testStoragePolicyProtoNoRetention = policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: (10 * time.Second).Nanoseconds(), Precision: time.Second.Nanoseconds(), }, } testStoragePolicyProtoBadPrecision = policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: (10 * time.Second).Nanoseconds(), Precision: 2, }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: time.Hour.Nanoseconds(), }, } @@ -407,11 +407,11 @@ func TestNewStoragePolicyFromProto(t *testing.T) { }{ { s: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -419,11 +419,11 @@ func TestNewStoragePolicyFromProto(t *testing.T) { }, { s: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(240 * time.Hour), }, }, diff --git a/src/metrics/rules/convert_test.go b/src/metrics/rules/convert_test.go index d5c595a273..3f6512e426 100644 --- a/src/metrics/rules/convert_test.go +++ b/src/metrics/rules/convert_test.go @@ -52,8 +52,8 @@ func TestToAggregationIDAndStoragePoliciesInvalidStoragePolicyProto(t *testing.T policiesProto := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{Precision: 1234}, - Retention: &policypb.Retention{Period: 5678}, + Resolution: policypb.Resolution{Precision: 1234}, + Retention: policypb.Retention{Period: 5678}, }, }, } @@ -65,11 +65,11 @@ func TestToAggregationIDAndStoragePoliciesInvalidAggregationTypes(t *testing.T) policiesProto := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -84,11 +84,11 @@ func TestToAggregationIDAndStoragePoliciesInconsistentAggregationIDs(t *testing. policiesProto := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -96,11 +96,11 @@ func TestToAggregationIDAndStoragePoliciesInconsistentAggregationIDs(t *testing. }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, @@ -116,33 +116,33 @@ func TestToAggregationIDAndStoragePoliciesDefaultAggregationID(t *testing.T) { policiesProto := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -163,11 +163,11 @@ func TestToAggregationIDAndStoragePoliciesCustomAggregationID(t *testing.T) { policiesProto := []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -175,11 +175,11 @@ func TestToAggregationIDAndStoragePoliciesCustomAggregationID(t *testing.T) { }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, @@ -187,11 +187,11 @@ func TestToAggregationIDAndStoragePoliciesCustomAggregationID(t *testing.T) { }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, diff --git a/src/metrics/rules/mapping_test.go b/src/metrics/rules/mapping_test.go index a714bcd3be..fb4e6620c2 100644 --- a/src/metrics/rules/mapping_test.go +++ b/src/metrics/rules/mapping_test.go @@ -51,11 +51,11 @@ var ( Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -73,11 +73,11 @@ var ( Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -87,11 +87,11 @@ var ( }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -113,29 +113,29 @@ var ( LastUpdatedBy: "someone", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -156,11 +156,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 1800 * time.Hour.Nanoseconds(), }, }, @@ -430,11 +430,11 @@ func TestNewMappingRuleSnapshotStoragePoliciesAndDropPolicy(t *testing.T) { proto := &rulepb.MappingRuleSnapshot{ StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, diff --git a/src/metrics/rules/rollup_target_test.go b/src/metrics/rules/rollup_target_test.go index 9cab2fab16..fab69e4dfc 100644 --- a/src/metrics/rules/rollup_target_test.go +++ b/src/metrics/rules/rollup_target_test.go @@ -48,11 +48,11 @@ func TestNewRollupTargetV1ProtoInvalidProto(t *testing.T) { Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -71,33 +71,33 @@ func TestNewRollupTargetV1ProtoWithDefaultAggregationID(t *testing.T) { Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -134,11 +134,11 @@ func TestNewRollupTargetV1ProtoWithCustomAggregationID(t *testing.T) { Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -146,11 +146,11 @@ func TestNewRollupTargetV1ProtoWithCustomAggregationID(t *testing.T) { }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, @@ -158,11 +158,11 @@ func TestNewRollupTargetV1ProtoWithCustomAggregationID(t *testing.T) { }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -229,8 +229,8 @@ func TestNewRollupTargetV2ProtoInvalidStoragePoliciesProto(t *testing.T) { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{Precision: 1234}, - Retention: &policypb.Retention{Period: 5678}, + Resolution: policypb.Resolution{Precision: 1234}, + Retention: policypb.Retention{Period: 5678}, }, }, } @@ -269,29 +269,29 @@ func TestNewRollupTargetV2Proto(t *testing.T) { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -472,29 +472,29 @@ func TestRollupTargetProto(t *testing.T) { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, diff --git a/src/metrics/rules/rollup_test.go b/src/metrics/rules/rollup_test.go index 3bbf8add9c..0a2655290a 100644 --- a/src/metrics/rules/rollup_test.go +++ b/src/metrics/rules/rollup_test.go @@ -60,11 +60,11 @@ var ( Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -88,11 +88,11 @@ var ( Policies: []*policypb.Policy{ &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -102,11 +102,11 @@ var ( }, &policypb.Policy{ StoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -157,29 +157,29 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Hour.Nanoseconds(), Precision: time.Hour.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 365 * 24 * time.Hour.Nanoseconds(), }, }, @@ -205,11 +205,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 720 * time.Hour.Nanoseconds(), }, }, @@ -243,11 +243,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 1800 * time.Hour.Nanoseconds(), }, }, @@ -490,11 +490,11 @@ func TestNewRollupRuleSnapshotFromV2ProtoInvalidProto(t *testing.T) { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 1800 * time.Hour.Nanoseconds(), }, }, diff --git a/src/metrics/rules/ruleset_test.go b/src/metrics/rules/ruleset_test.go index 1b4a4d5bc1..a1f55f1a92 100644 --- a/src/metrics/rules/ruleset_test.go +++ b/src/metrics/rules/ruleset_test.go @@ -1430,11 +1430,11 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 24 * time.Hour.Nanoseconds(), }, }, @@ -1448,29 +1448,29 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 6 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 5 * time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 48 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Minute.Nanoseconds(), Precision: time.Minute.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 48 * time.Hour.Nanoseconds(), }, }, @@ -1484,11 +1484,11 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 30 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 6 * time.Hour.Nanoseconds(), }, }, @@ -1507,11 +1507,11 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 12 * time.Hour.Nanoseconds(), }, }, @@ -1528,20 +1528,20 @@ func testMappingRulesConfig() []*rulepb.MappingRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: 10 * time.Second.Nanoseconds(), Precision: time.Second.Nanoseconds(), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: 2 * time.Hour.Nanoseconds(), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -1558,20 +1558,20 @@ func testMappingRulesConfig() []*rulepb.MappingRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(2 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -1593,20 +1593,20 @@ func testMappingRulesConfig() []*rulepb.MappingRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(12 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -1620,20 +1620,20 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(2 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -1655,11 +1655,11 @@ func testMappingRulesConfig() []*rulepb.MappingRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -1680,11 +1680,11 @@ func testMappingRulesConfig() []*rulepb.MappingRule { Filter: "mtagName1:mtagValue1", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -1721,11 +1721,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -1753,29 +1753,29 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(6 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -1803,11 +1803,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(30 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(6 * time.Hour), }, }, @@ -1840,11 +1840,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(12 * time.Hour), }, }, @@ -1872,20 +1872,20 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(2 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -1913,11 +1913,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -1950,29 +1950,29 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(12 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -1992,11 +1992,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -2024,20 +2024,20 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(2 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -2070,11 +2070,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, @@ -2107,11 +2107,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Minute), }, }, @@ -2144,11 +2144,11 @@ func testRollupRulesConfig() []*rulepb.RollupRule { }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(time.Hour), }, }, diff --git a/src/metrics/rules/store/kv/store_test.go b/src/metrics/rules/store/kv/store_test.go index fa1551131e..0286840125 100644 --- a/src/metrics/rules/store/kv/store_test.go +++ b/src/metrics/rules/store/kv/store_test.go @@ -96,11 +96,11 @@ var ( Filter: "tag1:value1 tag2:value2", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -119,20 +119,20 @@ var ( Filter: "tag3:value3 tag4:value4", StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -159,11 +159,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -202,11 +202,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -237,20 +237,20 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -283,11 +283,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -318,20 +318,20 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(5 * time.Minute), Precision: int64(time.Minute), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(48 * time.Hour), }, }, @@ -365,11 +365,11 @@ var ( }, StoragePolicies: []*policypb.StoragePolicy{ &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(10 * time.Second), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, diff --git a/src/query/remote/codecs_test.go b/src/query/remote/codecs_test.go index 03dcc49006..fc3ea85932 100644 --- a/src/query/remote/codecs_test.go +++ b/src/query/remote/codecs_test.go @@ -261,11 +261,11 @@ func TestNewRestrictQueryOptionsFromProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -312,11 +312,11 @@ func TestNewRestrictQueryOptionsFromProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, @@ -329,7 +329,7 @@ func TestNewRestrictQueryOptionsFromProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: -1, }, }, @@ -342,7 +342,7 @@ func TestNewRestrictQueryOptionsFromProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(-1), }, @@ -356,11 +356,11 @@ func TestNewRestrictQueryOptionsFromProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(-1), }, }, @@ -450,11 +450,11 @@ func TestRestrictQueryOptionsProto(t *testing.T) { RestrictQueryType: &rpcpb.RestrictQueryType{ MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ + Resolution: policypb.Resolution{ WindowSize: int64(time.Minute), Precision: int64(time.Second), }, - Retention: &policypb.Retention{ + Retention: policypb.Retention{ Period: int64(24 * time.Hour), }, }, From 6ed1c25225e1a7584595bd495fcc9a10b029840a Mon Sep 17 00:00:00 2001 From: Matt Way Date: Thu, 29 Oct 2020 23:30:00 -0400 Subject: [PATCH 09/47] Update issue templates --- .github/ISSUE_TEMPLATE/breaking-change.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/breaking-change.md diff --git a/.github/ISSUE_TEMPLATE/breaking-change.md b/.github/ISSUE_TEMPLATE/breaking-change.md new file mode 100644 index 0000000000..61fd9d1f3d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/breaking-change.md @@ -0,0 +1,20 @@ +--- +name: Breaking Change +about: Tell us about a breaking change +title: '' +labels: Breaking Change +assignees: '' + +--- + +**Expected Behavior** +A clear and concise description of what you expected to happen. + +**Observed Behavior** +A clear and concise description of what actually happened. + +**Steps To Reproduce** +How can we reproduce the issue? If applicable, please include sample configuration, commands, or other relevant context. Do not include any proprietary information. + +**Additional Context** +Add any other context about the observed change here. From 22be97537bc23cb400773554f8704216e4b87fef Mon Sep 17 00:00:00 2001 From: Matt Way Date: Thu, 29 Oct 2020 23:31:21 -0400 Subject: [PATCH 10/47] Update issue templates --- .github/ISSUE_TEMPLATE/breaking-change.md | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/breaking-change.md diff --git a/.github/ISSUE_TEMPLATE/breaking-change.md b/.github/ISSUE_TEMPLATE/breaking-change.md deleted file mode 100644 index 61fd9d1f3d..0000000000 --- a/.github/ISSUE_TEMPLATE/breaking-change.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Breaking Change -about: Tell us about a breaking change -title: '' -labels: Breaking Change -assignees: '' - ---- - -**Expected Behavior** -A clear and concise description of what you expected to happen. - -**Observed Behavior** -A clear and concise description of what actually happened. - -**Steps To Reproduce** -How can we reproduce the issue? If applicable, please include sample configuration, commands, or other relevant context. Do not include any proprietary information. - -**Additional Context** -Add any other context about the observed change here. From 3350a60fd25322756dbe977fdf150e1c3ed3eb1c Mon Sep 17 00:00:00 2001 From: Matt Way Date: Fri, 30 Oct 2020 00:00:55 -0400 Subject: [PATCH 11/47] Remove old code: m3nsch (#2822) --- .codecov.yml | 9 - .fossa.yml | 12 - Makefile | 3 - docker/m3nsch/Dockerfile | 25 - go.mod | 2 - kube/README.md | 45 - kube/m3nsch.yaml | 146 -- kube/scripts/m3nsch_client.sh | 54 - scripts/process-cover.sh | 5 +- src/cmd/services/m3nsch_client/cmd/init.go | 130 - src/cmd/services/m3nsch_client/cmd/m3nsch.go | 69 - src/cmd/services/m3nsch_client/cmd/modify.go | 82 - src/cmd/services/m3nsch_client/cmd/start.go | 67 - src/cmd/services/m3nsch_client/cmd/status.go | 74 - src/cmd/services/m3nsch_client/cmd/stop.go | 67 - src/cmd/services/m3nsch_client/cmd/util.go | 76 - src/cmd/services/m3nsch_client/main/main.go | 29 - .../services/m3nsch_server/config/config.go | 55 - src/cmd/services/m3nsch_server/main/main.go | 150 -- .../m3nsch_server/services/grpc_service.go | 113 - src/cmd/services/m3nsch_server/tcp/tcp.go | 54 - src/m3nsch/README.md | 107 - src/m3nsch/agent/agent.go | 374 --- src/m3nsch/agent/agent_options.go | 105 - src/m3nsch/agent/agent_test.go | 340 --- src/m3nsch/coordinator/coordinator.go | 434 ---- src/m3nsch/coordinator/coordinator_options.go | 77 - src/m3nsch/coordinator/coordinator_test.go | 150 -- src/m3nsch/datums/datum.go | 79 - src/m3nsch/datums/registry.go | 74 - src/m3nsch/datums/types.go | 55 - src/m3nsch/examples/sample_workload.go | 147 -- src/m3nsch/generated/convert/to_api.go | 64 - src/m3nsch/generated/convert/to_proto.go | 56 - src/m3nsch/generated/mocks/generate.go | 23 - src/m3nsch/generated/proto/generate.go | 23 - .../generated/proto/m3nsch/m3nsch.pb.go | 2099 ----------------- .../generated/proto/m3nsch/m3nsch.proto | 68 - .../generated/proto/m3nsch/m3nsch_pb_mock.go | 254 -- src/m3nsch/types.go | 207 -- 40 files changed, 2 insertions(+), 6001 deletions(-) delete mode 100644 docker/m3nsch/Dockerfile delete mode 100644 kube/m3nsch.yaml delete mode 100755 kube/scripts/m3nsch_client.sh delete mode 100644 src/cmd/services/m3nsch_client/cmd/init.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/m3nsch.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/modify.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/start.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/status.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/stop.go delete mode 100644 src/cmd/services/m3nsch_client/cmd/util.go delete mode 100644 src/cmd/services/m3nsch_client/main/main.go delete mode 100644 src/cmd/services/m3nsch_server/config/config.go delete mode 100644 src/cmd/services/m3nsch_server/main/main.go delete mode 100644 src/cmd/services/m3nsch_server/services/grpc_service.go delete mode 100644 src/cmd/services/m3nsch_server/tcp/tcp.go delete mode 100644 src/m3nsch/README.md delete mode 100644 src/m3nsch/agent/agent.go delete mode 100644 src/m3nsch/agent/agent_options.go delete mode 100644 src/m3nsch/agent/agent_test.go delete mode 100644 src/m3nsch/coordinator/coordinator.go delete mode 100644 src/m3nsch/coordinator/coordinator_options.go delete mode 100644 src/m3nsch/coordinator/coordinator_test.go delete mode 100644 src/m3nsch/datums/datum.go delete mode 100644 src/m3nsch/datums/registry.go delete mode 100644 src/m3nsch/datums/types.go delete mode 100644 src/m3nsch/examples/sample_workload.go delete mode 100644 src/m3nsch/generated/convert/to_api.go delete mode 100644 src/m3nsch/generated/convert/to_proto.go delete mode 100644 src/m3nsch/generated/mocks/generate.go delete mode 100644 src/m3nsch/generated/proto/generate.go delete mode 100644 src/m3nsch/generated/proto/m3nsch/m3nsch.pb.go delete mode 100644 src/m3nsch/generated/proto/m3nsch/m3nsch.proto delete mode 100644 src/m3nsch/generated/proto/m3nsch/m3nsch_pb_mock.go delete mode 100644 src/m3nsch/types.go diff --git a/.codecov.yml b/.codecov.yml index 31dadc9239..329a94e74f 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -44,10 +44,6 @@ coverage: target: auto threshold: 5% flags: m3ninx - m3nsch: - target: auto - threshold: 5% - flags: m3nsch m3em: target: auto threshold: 5% @@ -79,11 +75,6 @@ coverage: m3ninx: paths: - src/m3ninx/ - m3nsch: - paths: - - src/m3nsch/ - - src/cmd/services/m3nsch_server - - src/cmd/services/m3nsch_client m3em: paths: - src/m3em/ diff --git a/.fossa.yml b/.fossa.yml index 45a9f148c6..f9a41210d6 100755 --- a/.fossa.yml +++ b/.fossa.yml @@ -60,18 +60,6 @@ analyze: path: src/cmd/services/m3em_agent/main options: allow-unresolved: true - - name: github.com/m3db/m3/src/cmd/services/m3nsch_client/main - type: go - target: github.com/m3db/m3/src/cmd/services/m3nsch_client/main - path: src/cmd/services/m3nsch_client/main - options: - allow-unresolved: true - - name: github.com/m3db/m3/src/cmd/services/m3nsch_server/main - type: go - target: github.com/m3db/m3/src/cmd/services/m3nsch_server/main - path: src/cmd/services/m3nsch_server/main - options: - allow-unresolved: true - name: github.com/m3db/m3/src/cmd/services/m3query/main type: go target: github.com/m3db/m3/src/cmd/services/m3query/main diff --git a/Makefile b/Makefile index 8a4031a3aa..e208a0ab93 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,6 @@ SERVICES := \ m3query \ m3collector \ m3em_agent \ - m3nsch_server \ - m3nsch_client \ m3comparator \ r2ctl \ @@ -78,7 +76,6 @@ SUBDIRS := \ dbnode \ query \ m3em \ - m3nsch \ m3ninx \ aggregator \ ctl \ diff --git a/docker/m3nsch/Dockerfile b/docker/m3nsch/Dockerfile deleted file mode 100644 index 92101f109e..0000000000 --- a/docker/m3nsch/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# stage 1: build -FROM golang:1.13-alpine3.11 AS builder -LABEL maintainer="The M3DB Authors " - -# Install deps -RUN apk add --update git make bash - -# Add source code -RUN mkdir -p /go/src/github.com/m3db/m3 -ADD . /go/src/github.com/m3db/m3 - -# Build m3nsch binary -RUN cd /go/src/github.com/m3db/m3/ && \ - git submodule update --init && \ - make m3nsch_server-linux-amd64 && \ - make m3nsch_client-linux-amd64 - -# stage 2: lightweight "release" -FROM alpine:3.11 -LABEL maintainer="The M3DB Authors " - -COPY --from=builder /go/src/github.com/m3db/m3/bin/m3nsch_server /bin/ -COPY --from=builder /go/src/github.com/m3db/m3/bin/m3nsch_client /bin/ - -ENTRYPOINT [ "/bin/m3nsch_server" ] diff --git a/go.mod b/go.mod index 5201da3ec1..91e3ee308a 100644 --- a/go.mod +++ b/go.mod @@ -93,8 +93,6 @@ require ( github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.1 github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect github.com/stretchr/testify v1.6.1 github.com/subosito/gotenv v1.2.1-0.20190917103637-de67a6614a4d // indirect diff --git a/kube/README.md b/kube/README.md index 50c5ad8049..74fad06a0a 100644 --- a/kube/README.md +++ b/kube/README.md @@ -8,48 +8,3 @@ This doc is aimed at developers building M3DB on Kubernetes. End users should se In order to make it possible to set up m3db using a single YAML file that can be passed as a URL to `kubectl apply`, we bundle our manifests into a single `bundle.yaml` file. In order to create a bundle, simply run `build_bundle.sh`. It will take care of ordering (i.e. `Namespace` object comes first) and produce a single YAML file. - -## `m3nsch` - -A Kubernetes manifests for running `m3nsch` is included. `m3nsch` is a tool for load-testing M3DB clusters. The manifest -will configure `m3nsch` to generate load for a cluster created according to `bundle.yaml`. The `m3nsch` setup will -contain 2 deployments: 1 for the agent which generates and sends load and 1 for the client from which you can control -the agent. For example: - -``` -$ kubectl apply -f kube/m3nsch.yaml -configmap/m3nsch-server-config created -service/m3nsch-agent created -service/m3nsch-agent-debug created -deployment.apps/m3nsch-server created -deployment.apps/m3nsch-client created - -$ ./kube/scripts/m3nsch_client.sh init -t foo -z default_zone -v default_env -n m3db-cluster -... -2018/09/30 15:24:16 Go Runtime version: go1.10.2 -2018/09/30 15:24:16 Build Version: v0.4.5 -2018/09/30 15:24:16 Build Revision: aa253ca01 -2018/09/30 15:24:16 Build Branch: schallert/m3nsch_update -2018/09/30 15:24:16 Build Date: 2018-09-29-21:31:30 -2018/09/30 15:24:16 Build TimeUnix: 1538256690 - -$ ./kube/scripts/m3nsch_client.sh start -... -2018/09/30 15:24:23 Go Runtime version: go1.10.2 -2018/09/30 15:24:23 Build Version: v0.4.5 -2018/09/30 15:24:23 Build Revision: aa253ca01 -2018/09/30 15:24:23 Build Branch: schallert/m3nsch_update -2018/09/30 15:24:23 Build Date: 2018-09-29-21:31:30 -2018/09/30 15:24:23 Build TimeUnix: 1538256690 -15:24:23.205737[I] workload started! -``` - -You can view the stats of the ongoing benchmark via `m3nsch_server`'s prometheus endpoints, or using our [grafana dashboard][dash]: -``` -$ kubectl port-forward svc/m3nch-agent 12580 -$ curl -sSf localhost:12580/metrics | rg write.+success -# TYPE write_req_success counter -write_req_success 146000 -``` - -[dash]: https://raw.githubusercontent.com/m3db/m3/master/integrations/grafana/m3nsch_dashboard.json diff --git a/kube/m3nsch.yaml b/kube/m3nsch.yaml deleted file mode 100644 index 27e4a6b40a..0000000000 --- a/kube/m3nsch.yaml +++ /dev/null @@ -1,146 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: m3nsch-server-config -data: - m3nsch_server.yaml: |+ - server: - listenAddress: 0.0.0.0:2580 - debugAddress: 0.0.0.0:12580 - cpuFactor: 0.9 - - metrics: - prometheus: - handlerPath: /metrics - samplingRate: 1.0 - extended: detailed - sanitization: prometheus - - m3nsch: - concurrency: 2000 - numPointsPerDatum: 60 - - dbClient: - config: - service: - zone: default_zone - env: default_env - service: m3db - etcdClusters: - - zone: default_zone - endpoints: - - etcd-0.etcd:2379 - - etcd-1.etcd:2379 - - etcd-2.etcd:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 ---- -apiVersion: v1 -kind: Service -metadata: - name: m3nsch-agent - labels: - app: m3nsch - component: agent -spec: - type: ClusterIP - clusterIP: None - ports: - - name: agent - port: 2580 - - name: debug - port: 12580 - selector: - app: m3nsch - component: agent ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: m3nsch-agent - labels: - app: m3nsch - component: agent -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 100% - selector: - matchLabels: - app: m3nsch - component: agent - template: - metadata: - labels: - app: m3nsch - component: agent - spec: - containers: - - name: m3nsch-agent - image: quay.io/m3/m3nsch:latest - args: - - "-f" - - "/etc/m3nsch/m3nsch_server.yaml" - volumeMounts: - - name: server-config - mountPath: /etc/m3nsch/ - imagePullPolicy: Always - ports: - - name: agent - containerPort: 2580 - protocol: TCP - - name: debug - containerPort: 12580 - protocol: TCP - volumes: - - name: server-config - configMap: - name: m3nsch-server-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: m3nsch-client - labels: - app: m3nsch - component: client -spec: - replicas: 1 - selector: - matchLabels: - app: m3nsch - component: client - template: - metadata: - labels: - app: m3nsch - component: client - spec: - containers: - - name: m3nsch-client - image: quay.io/m3/m3nsch:latest - command: - - "/bin/ash" - args: ["-c", "while true; do sleep 300; done"] - imagePullPolicy: Always - # If you're running in a shell it may take a while to kill - terminationGracePeriodSeconds: 5 diff --git a/kube/scripts/m3nsch_client.sh b/kube/scripts/m3nsch_client.sh deleted file mode 100755 index 9911361645..0000000000 --- a/kube/scripts/m3nsch_client.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# The script makes it easier to run m3nsch by gathering the endpoints of all -# agents and constructing the base m3nsch_client command. You may provide extra -# labels to target agent pods if you are running multiple m3nsch agent -# deployments in the same namespace (i.e. for concurrent load tests against -# multiple m3db clusters). -# -# Example usage: -# ./m3nsch_client.sh init -t foo -z default_zone -v default_env -n metrics-10s:2d -c 50000 -i 5000 -u 0.5 -# LABELS="deployment=foo" ./m3nsch_client.sh init -t foo -z default_zone -v default_env -n metrics-10s:2d -c 50000 -i 5000 -u 0.5 -# ./m3nsch_client.sh start -# ./m3nsch_client.sh stop - -set -eo pipefail - -WARMUP=${WARMUP:-10} - -# Get endpoints of the agents -function get_endpoints() { - local selector="app=m3nsch,component=agent" - if [[ -n "$LABELS" ]]; then - selector="$selector,$LABELS" - fi - local jsonpath='{range .items[*]}{.status.podIP}:{.spec.containers[0].ports[0].containerPort},{end}' - # cut trailing comma - kubectl get po -l "$selector" -o jsonpath="$jsonpath" | sed 's/,$//' -} - -CLIENT_POD=$(kubectl get po | grep client | awk '{print $1}') - -if [[ -z "$CLIENT_POD" ]]; then - echo "could not find client pod" - exit 1 -fi - -AGENT_ENDPOINTS=$(get_endpoints) - -if [[ "$#" -eq 1 && "$1" == "start" ]]; then - ( - IFS="," - for EP in $AGENT_ENDPOINTS; do - set -x - kubectl exec "$CLIENT_POD" -- ./bin/m3nsch_client -e "$EP" "$@" - sleep "$WARMUP" - set +x - done - ) - exit -fi - -set -x -kubectl exec "$CLIENT_POD" -- ./bin/m3nsch_client -e "$AGENT_ENDPOINTS" "$@" -set +x diff --git a/scripts/process-cover.sh b/scripts/process-cover.sh index 21f0c0d45c..8e6ea0a9c0 100755 --- a/scripts/process-cover.sh +++ b/scripts/process-cover.sh @@ -8,7 +8,7 @@ fi COVERFILE=$1 SUBMIT_COVER="$(dirname $0)/../.ci/codecov.sh" -TARGETS=("aggregator" "dbnode" "query" "collector" "cluster" "m3ninx" "m3nsch" "m3em" "x") +TARGETS=("aggregator" "dbnode" "query" "collector" "cluster" "m3ninx" "m3em" "x") target_patterns() { case $1 in 'collector') echo "^mode|github.com/m3db/m3/src/collector|github.com/m3db/m3/src/cmd/services/m3collector";; @@ -18,7 +18,6 @@ target_patterns() { 'query') echo "^mode|github.com/m3db/m3/src/query|github.com/m3db/m3/src/cmd/services/m3query";; 'm3em') echo "^mode|github.com/m3db/m3/src/m3em|github.com/m3db/m3/src/cmd/services/m3em_agent";; 'm3ninx') echo "^mode|github.com/m3db/m3/src/m3ninx";; - 'm3nsch') echo "^mode|github.com/m3db/m3/src/m3nsch";; 'x') echo "^mode|github.com/m3db/m3/src/x";; 'msg') echo "^mode|github.com/m3db/m3/src/msg";; *) echo "unknown key: $1"; exit 1;; @@ -33,4 +32,4 @@ fi for t in ${TARGETS[@]}; do cat $COVERFILE | grep -E $(target_patterns $t) > ${t}.out ${SUBMIT_COVER} -f ${t}.out -F ${t} -done \ No newline at end of file +done diff --git a/src/cmd/services/m3nsch_client/cmd/init.go b/src/cmd/services/m3nsch_client/cmd/init.go deleted file mode 100644 index d2f839b866..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/init.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "fmt" - "log" - - "github.com/m3db/m3/src/m3nsch/coordinator" - xerrors "github.com/m3db/m3/src/x/errors" - "github.com/m3db/m3/src/x/instrument" - - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var ( - // local flags - localInitFlags initFlags - - // InitCmd represents the base command when called without any subcommands - initCmd *cobra.Command - - defaultEnv = "test" - defaultZone = "sjc1" -) - -func init() { - initCmd = &cobra.Command{ - Use: "init", - Short: "Initialize agent processes", - Long: "Initialize agent processes with any resources required to execute workload", - Run: initExec, - Example: `# Initialize agents with default workload: -./m3nsch_client -e ",...,:" init -t any_string_breadcrumb - -# Initialize agents with explicit workload: -./m3nsch_client --endpoints ",...,:" init \ - --token any_string_breadcrumb \ - --target-zone sjc1 \ - --target-env prod \ - --metric-prefix m3nsch_metric_prefix \ - --namespace testmetrics \ - --cardinality 1000000 \ - --ingress-qps 200000 \`, - } - - flags := initCmd.Flags() - flags.StringVarP(&localInitFlags.token, "token", "t", "", - `[required] unique identifier required for all subsequent interactions on this workload`) - flags.BoolVarP(&localInitFlags.force, "force", "f", false, - `force initialization, stop any running workload`) - flags.StringVarP(&localInitFlags.targetZone, "target-zone", "z", defaultZone, - `target zone for load test`) - flags.StringVarP(&localInitFlags.targetEnv, "target-env", "v", defaultEnv, - `target env for load test`) - registerWorkloadFlags(flags, &localInitFlags.workload) -} - -type initFlags struct { - token string - force bool - workload cliWorkload - targetZone string - targetEnv string -} - -func (f initFlags) validate() error { - var multiErr xerrors.MultiError - if f.token == "" { - multiErr = multiErr.Add(fmt.Errorf("token is not set")) - } - if f.targetEnv == "" { - multiErr = multiErr.Add(fmt.Errorf("target-env is not set")) - } - if f.targetZone == "" { - multiErr = multiErr.Add(fmt.Errorf("target-zone is not set")) - } - if err := f.workload.validate(); err != nil { - multiErr = multiErr.Add(err) - } - return multiErr.FinalError() -} - -func initExec(cmd *cobra.Command, _ []string) { - if !gFlags.isValid() { - log.Fatalf("Invalid flags: %v", M3nschCmd.UsageString()) - } - if err := localInitFlags.validate(); err != nil { - log.Fatalf("Invalid flags: %v\n%s", err, cmd.UsageString()) - } - - var ( - workload = localInitFlags.workload.toM3nschWorkload() - iopts = instrument.NewOptions() - logger = iopts.Logger() - mOpts = coordinator.NewOptions(iopts) - coord, err = coordinator.New(mOpts, gFlags.endpoints) - ) - - if err != nil { - logger.Fatal("unable to create coord", zap.Error(err)) - } - defer coord.Teardown() - - err = coord.Init(localInitFlags.token, workload, localInitFlags.force, - localInitFlags.targetZone, localInitFlags.targetEnv) - - if err != nil { - logger.Fatal("unable to initialize", zap.Error(err)) - } -} diff --git a/src/cmd/services/m3nsch_client/cmd/m3nsch.go b/src/cmd/services/m3nsch_client/cmd/m3nsch.go deleted file mode 100644 index b271e83ae5..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/m3nsch.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" -) - -var ( - // globalFlags - gFlags globalFlags - - // M3nschCmd represents the base command when called without any subcommands - M3nschCmd = &cobra.Command{ - Use: "m3nsch_client", - Short: "CLI interface to m3nsch - M3DB load generation", - } -) - -type globalFlags struct { - endpoints []string -} - -func (f globalFlags) isValid() bool { - return len(f.endpoints) != 0 -} - -func init() { - flags := M3nschCmd.PersistentFlags() - flags.StringSliceVarP(&gFlags.endpoints, "endpoints", "e", []string{}, - `host:port for each of the agent process endpoints`) - - M3nschCmd.AddCommand( - statusCmd, - initCmd, - startCmd, - stopCmd, - modifyCmd, - ) -} - -// Run executes the m3nsch command. -func Run() { - if err := M3nschCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(-1) - } -} diff --git a/src/cmd/services/m3nsch_client/cmd/modify.go b/src/cmd/services/m3nsch_client/cmd/modify.go deleted file mode 100644 index a7309c7230..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/modify.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "log" - - "github.com/m3db/m3/src/m3nsch/coordinator" - "github.com/m3db/m3/src/x/instrument" - - "go.uber.org/zap" - "github.com/spf13/cobra" -) - -var ( - modifyWorkload cliWorkload - - modifyCmd = &cobra.Command{ - Use: "modify", - Short: "modify the workload used in load generation", - Long: "Modify changes the worload used during load generation on each agent process", - Run: modifyExec, - Example: `# Modify agents with explicit workload: -./m3nsch_client --endpoints ",...,:" modify \ - --metric-prefix m3nsch_metric_prefix \ - --namespace metrics \ - --cardinality 1000000 \ - --ingress-qps 200000 \`, - } -) - -func init() { - flags := modifyCmd.Flags() - registerWorkloadFlags(flags, &modifyWorkload) -} - -func modifyExec(cmd *cobra.Command, _ []string) { - if !gFlags.isValid() { - log.Fatalf("Invalid flags\n%s", cmd.UsageString()) - } - if err := modifyWorkload.validate(); err != nil { - log.Fatalf("Invalid flags: %v\n%s", err, cmd.UsageString()) - } - - var ( - workload = modifyWorkload.toM3nschWorkload() - iopts = instrument.NewOptions() - logger = iopts.Logger() - mOpts = coordinator.NewOptions(iopts) - coord, err = coordinator.New(mOpts, gFlags.endpoints) - ) - - if err != nil { - logger.Fatal("unable to create coord", zap.Error(err)) - } - defer coord.Teardown() - - err = coord.SetWorkload(workload) - if err != nil { - logger.Fatal("unable to modify workload", zap.Error(err)) - } - - logger.Info("workload modified successfully!") -} diff --git a/src/cmd/services/m3nsch_client/cmd/start.go b/src/cmd/services/m3nsch_client/cmd/start.go deleted file mode 100644 index 1eda4b5ca0..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/start.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "log" - - "github.com/m3db/m3/src/m3nsch/coordinator" - "github.com/m3db/m3/src/x/instrument" - - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var ( - startCmd = &cobra.Command{ - Use: "start", - Short: "start the load generation", - Long: "Start kicks off the load generation on each agent process", - Run: startExec, - Example: `# Start the load generation process on various agents: -./m3nsch_client -e ",...,:" start`, - } -) - -func startExec(_ *cobra.Command, _ []string) { - if !gFlags.isValid() { - log.Fatalf("unable to execute, invalid flags\n%s", M3nschCmd.UsageString()) - } - - var ( - iopts = instrument.NewOptions() - logger = iopts.Logger() - mOpts = coordinator.NewOptions(iopts) - coord, err = coordinator.New(mOpts, gFlags.endpoints) - ) - - if err != nil { - logger.Fatal("unable to create coordinator", zap.Error(err)) - } - defer coord.Teardown() - - err = coord.Start() - if err != nil { - logger.Fatal("unable to start workload", zap.Error(err)) - } - - logger.Info("workload started!") -} diff --git a/src/cmd/services/m3nsch_client/cmd/status.go b/src/cmd/services/m3nsch_client/cmd/status.go deleted file mode 100644 index 0b1c837c2e..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/status.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "log" - - "github.com/m3db/m3/src/m3nsch/coordinator" - "github.com/m3db/m3/src/x/instrument" - - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var ( - statusCmd = &cobra.Command{ - Use: "status", - Short: "Retrieves the status of agent processes", - Long: "Status retrieves a status from each of the described agent processes", - Run: statusExec, - Example: `# Get status from the various agents: -./m3nsch_client -e ",...,:" status`, - } -) - -func statusExec(_ *cobra.Command, _ []string) { - if !gFlags.isValid() { - log.Fatalf("unable to execute, invalid flags\n%s", M3nschCmd.UsageString()) - } - - var ( - iopts = instrument.NewOptions() - logger = iopts.Logger() - mOpts = coordinator.NewOptions(iopts) - coord, err = coordinator.New(mOpts, gFlags.endpoints) - ) - - if err != nil { - logger.Fatal("unable to create coordinator", zap.Error(err)) - } - defer coord.Teardown() - - statusMap, err := coord.Status() - if err != nil { - logger.Fatal("unable to retrieve status", zap.Error(err)) - } - - for endpoint, status := range statusMap { - token := status.Token - if token == "" { - token = "" - } - logger.Sugar().Infof("[%v] MaxQPS: %d, Status: %v, Token: %v, Workload: %+v", - endpoint, status.MaxQPS, status.Status, token, status.Workload) - } -} diff --git a/src/cmd/services/m3nsch_client/cmd/stop.go b/src/cmd/services/m3nsch_client/cmd/stop.go deleted file mode 100644 index b0e653b2e3..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/stop.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "log" - - "github.com/m3db/m3/src/m3nsch/coordinator" - "github.com/m3db/m3/src/x/instrument" - - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var ( - stopCmd = &cobra.Command{ - Use: "stop", - Short: "stop the load generation", - Long: "Stop stops the load generation on each agent process", - Run: stopExec, - Example: `# Stop the load generation process on various agents: -./m3nsch_client -e ",...,:" stop`, - } -) - -func stopExec(_ *cobra.Command, _ []string) { - if !gFlags.isValid() { - log.Fatalf("unable to execute, invalid flags\n%s", M3nschCmd.UsageString()) - } - - var ( - iopts = instrument.NewOptions() - logger = iopts.Logger() - mOpts = coordinator.NewOptions(iopts) - coordinator, err = coordinator.New(mOpts, gFlags.endpoints) - ) - - if err != nil { - logger.Fatal("unable to create coordinator", zap.Error(err)) - } - defer coordinator.Teardown() - - err = coordinator.Stop() - if err != nil { - logger.Fatal("unable to stop workload", zap.Error(err)) - } - - logger.Info("workload stopped!") -} diff --git a/src/cmd/services/m3nsch_client/cmd/util.go b/src/cmd/services/m3nsch_client/cmd/util.go deleted file mode 100644 index f56640a369..0000000000 --- a/src/cmd/services/m3nsch_client/cmd/util.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - "fmt" - "time" - - "github.com/m3db/m3/src/m3nsch" - xerrors "github.com/m3db/m3/src/x/errors" - - "github.com/spf13/pflag" -) - -type cliWorkload struct { - m3nsch.Workload - baseTimeOffset time.Duration -} - -func (w *cliWorkload) validate() error { - var multiErr xerrors.MultiError - if w.baseTimeOffset >= 0 { - multiErr = multiErr.Add(fmt.Errorf("basetime-offset must be negative")) - } - if w.Cardinality <= 0 { - multiErr = multiErr.Add(fmt.Errorf("cardinality must be a positive integer")) - } - if w.IngressQPS <= 0 { - multiErr = multiErr.Add(fmt.Errorf("ingress-qps must be a positive integer")) - } - if w.Namespace == "" { - multiErr = multiErr.Add(fmt.Errorf("namespace must be set")) - } - if w.UniqueAmplifier < 0.0 || w.UniqueAmplifier > 1.0 { - multiErr = multiErr.Add(fmt.Errorf("unique-amplifier must be between 0.0 and 1.0 (is %f)", w.UniqueAmplifier)) - } - return multiErr.FinalError() -} - -func (w *cliWorkload) toM3nschWorkload() m3nsch.Workload { - w.BaseTime = time.Now().Add(w.baseTimeOffset) - return w.Workload -} - -func registerWorkloadFlags(flags *pflag.FlagSet, workload *cliWorkload) { - flags.DurationVarP(&workload.baseTimeOffset, "basetime-offset", "b", -2*time.Minute, - `offset from current time to use for load, e.g. -2m, -30s`) - flags.StringVarP(&workload.MetricPrefix, "metric-prefix", "p", "m3nsch_", - `prefix added to each metric`) - flags.StringVarP(&workload.Namespace, "namespace", "n", "testmetrics", - `target namespace`) - flags.IntVarP(&workload.Cardinality, "cardinality", "c", 10000, - `aggregate workload cardinality`) - flags.IntVarP(&workload.IngressQPS, "ingress-qps", "i", 1000, - `aggregate workload ingress qps`) - flags.Float64VarP(&workload.UniqueAmplifier, "unique-amplifier", "u", 0.0, - `% of generatic metrics as float [0.0,1.0] that will be unique`) -} diff --git a/src/cmd/services/m3nsch_client/main/main.go b/src/cmd/services/m3nsch_client/main/main.go deleted file mode 100644 index b134e0c824..0000000000 --- a/src/cmd/services/m3nsch_client/main/main.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -import ( - m3nsch "github.com/m3db/m3/src/cmd/services/m3nsch_client/cmd" -) - -func main() { - m3nsch.Run() -} diff --git a/src/cmd/services/m3nsch_server/config/config.go b/src/cmd/services/m3nsch_server/config/config.go deleted file mode 100644 index 28f1854e9d..0000000000 --- a/src/cmd/services/m3nsch_server/config/config.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package config - -import ( - "github.com/m3db/m3/src/dbnode/client" - xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/instrument" -) - -// Configuration represents the knobs available to configure a m3nsch_server -type Configuration struct { - Server ServerConfiguration `yaml:"server"` - M3nsch M3nschConfiguration `yaml:"m3nsch" validate:"nonzero"` - Metrics instrument.MetricsConfiguration `yaml:"metrics" validate:"nonzero"` - DBClient client.Configuration `yaml:"dbClient"` -} - -// ServerConfiguration represents the knobs available to configure server properties -type ServerConfiguration struct { - ListenAddress string `yaml:"listenAddress" validate:"nonzero"` - DebugAddress string `yaml:"debugAddress" validate:"nonzero"` - CPUFactor float64 `yaml:"cpuFactor" validate:"min=0.5,max=3.0"` -} - -// M3nschConfiguration represents the knobs available to configure m3nsch properties -type M3nschConfiguration struct { - Concurrency int `yaml:"concurrency" validate:"min=500,max=5000"` - NumPointsPerDatum int `yaml:"numPointsPerDatum" validate:"min=10,max=120"` -} - -// New returns a Configuration read from the specified path -func New(filename string) (Configuration, error) { - var conf Configuration - err := xconfig.LoadFile(&conf, filename, xconfig.Options{}) - return conf, err -} diff --git a/src/cmd/services/m3nsch_server/main/main.go b/src/cmd/services/m3nsch_server/main/main.go deleted file mode 100644 index 79eeba0012..0000000000 --- a/src/cmd/services/m3nsch_server/main/main.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -import ( - "log" - "net" - "net/http" - _ "net/http/pprof" - "runtime" - "time" - - "github.com/m3db/m3/src/cmd/services/m3nsch_server/config" - "github.com/m3db/m3/src/cmd/services/m3nsch_server/services" - "github.com/m3db/m3/src/cmd/services/m3nsch_server/tcp" - "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/agent" - "github.com/m3db/m3/src/m3nsch/datums" - proto "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch" - xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/instrument" - - "github.com/pborman/getopt" - "github.com/uber-go/tally" - "go.uber.org/zap" - "google.golang.org/grpc" -) - -func main() { - var ( - configFile = getopt.StringLong("config-file", 'f', "", "Configuration file") - ) - getopt.Parse() - if len(*configFile) == 0 { - getopt.Usage() - return - } - - rawLogger, err := zap.NewProduction() - if err != nil { - log.Fatalf("unable to create logger: %v", err) - } - - logger := rawLogger.Sugar() - conf, err := config.New(*configFile) - if err != nil { - logger.Fatalf("unable to read configuration file: %v", err.Error()) - } - - xconfig.WarnOnDeprecation(conf, rawLogger) - - maxProcs := int(float64(runtime.NumCPU()) * conf.Server.CPUFactor) - logger.Infof("setting maxProcs = %d", maxProcs) - runtime.GOMAXPROCS(maxProcs) - - StartPProfServer(conf.Server.DebugAddress, rawLogger) - - scope, closer, err := conf.Metrics.NewRootScope() - if err != nil { - logger.Fatalf("could not connect to metrics: %v", err) - } - defer closer.Close() - - listener, err := tcp.NewTCPListener(conf.Server.ListenAddress, 3*time.Minute) - if err != nil { - logger.Fatalf("could not create TCP Listener: %v", err) - } - - iopts := instrument. - NewOptions(). - SetLogger(rawLogger). - SetMetricsScope(scope). - SetTimerOptions(instrument.TimerOptions{StandardSampleRate: conf.Metrics.SampleRate()}) - datumRegistry := datums.NewDefaultRegistry(conf.M3nsch.NumPointsPerDatum) - agentOpts := agent.NewOptions(iopts). - SetConcurrency(conf.M3nsch.Concurrency). - SetNewSessionFn(newSessionFn(conf, iopts)) - agent := agent.New(datumRegistry, agentOpts) - ServeGRPCService(listener, agent, scope, rawLogger) -} - -func newSessionFn(conf config.Configuration, iopts instrument.Options) m3nsch.NewSessionFn { - return m3nsch.NewSessionFn(func(zone, env string) (client.Session, error) { - cluster, err := conf.DBClient.EnvironmentConfig.Services.SyncCluster() - if err != nil { - return nil, err - } - - svc := cluster.Service - svc.Env = env - svc.Zone = zone - cl, err := conf.DBClient.NewClient(client.ConfigurationParameters{ - InstrumentOptions: iopts, - }) - if err != nil { - return nil, err - } - return cl.NewSession() - }) -} - -// StartPProfServer starts a pprof server at specified address, or crashes -// the program on failure. -func StartPProfServer(debugAddress string, logger *zap.Logger) { - go func() { - if err := http.ListenAndServe(debugAddress, nil); err != nil { - logger.Fatal("unable to serve debug server", zap.Error(err)) - } - }() - logger.Info("serving pprof endpoints", zap.String("address", debugAddress)) -} - -// ServeGRPCService serves m3nsch_server GRPC service at the specified address. -func ServeGRPCService( - listener net.Listener, - agent m3nsch.Agent, - scope tally.Scope, - logger *zap.Logger, -) { - server := grpc.NewServer(grpc.MaxConcurrentStreams(16384)) - service, err := services.NewGRPCService(agent, scope, logger) - if err != nil { - logger.Fatal("could not create grpc service", zap.Error(err)) - } - proto.RegisterMenschServer(server, service) - logger.Info("serving m3nsch endpoints", zap.Stringer("address", listener.Addr())) - err = server.Serve(listener) - if err != nil { - logger.Fatal("could not serve", zap.Error(err)) - } -} diff --git a/src/cmd/services/m3nsch_server/services/grpc_service.go b/src/cmd/services/m3nsch_server/services/grpc_service.go deleted file mode 100644 index e917faff22..0000000000 --- a/src/cmd/services/m3nsch_server/services/grpc_service.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package services - -import ( - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/generated/convert" - proto "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch" - - "github.com/uber-go/tally" - "go.uber.org/zap" - "golang.org/x/net/context" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" -) - -// NewGRPCService returns a new GRPCService wrapping a m3nsch.Agent -func NewGRPCService( - agent m3nsch.Agent, - metricsScope tally.Scope, - logger *zap.Logger, -) (proto.MenschServer, error) { - return &menschServer{ - agent: agent, - scope: metricsScope.SubScope("grpc-server"), - logger: logger, - }, nil -} - -type menschServer struct { - scope tally.Scope - logger *zap.Logger - agent m3nsch.Agent -} - -func (ms *menschServer) Status(ctx context.Context, req *proto.StatusRequest) (*proto.StatusResponse, error) { - status := ms.agent.Status() - workload := convert.ToProtoWorkload(ms.agent.Workload()) - response := &proto.StatusResponse{ - Token: status.Token, - Status: convert.ToProtoStatus(status.Status), - MaxQPS: ms.agent.MaxQPS(), - Workload: &workload, - } - return response, nil -} - -func (ms *menschServer) Init(ctx context.Context, req *proto.InitRequest) (*proto.InitResponse, error) { - if req == nil { - return nil, grpc.Errorf(codes.InvalidArgument, "nil request") - } - - ms.logger.Sugar().Debugf("received init request: %v", req.String()) - workload, err := convert.ToM3nschWorkload(req.GetWorkload()) - if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "unable to parse workload: %v", err) - } - - err = ms.agent.Init(req.GetToken(), workload, req.GetForce(), - req.GetTargetZone(), req.GetTargetEnv()) - if err != nil { - return nil, grpc.Errorf(codes.Unavailable, err.Error()) - } - - return &proto.InitResponse{}, nil -} - -func (ms *menschServer) Start(context.Context, *proto.StartRequest) (*proto.StartResponse, error) { - ms.logger.Sugar().Debugf("received Start() request") - if err := ms.agent.Start(); err != nil { - return nil, grpc.Errorf(codes.Unknown, err.Error()) - } - return &proto.StartResponse{}, nil -} - -func (ms *menschServer) Stop(context.Context, *proto.StopRequest) (*proto.StopResponse, error) { - ms.logger.Sugar().Debugf("received Stop() request") - if err := ms.agent.Stop(); err != nil { - return nil, grpc.Errorf(codes.Unknown, err.Error()) - } - return &proto.StopResponse{}, nil -} - -func (ms *menschServer) Modify(_ context.Context, req *proto.ModifyRequest) (*proto.ModifyResponse, error) { - ms.logger.Sugar().Debugf("received Modify() request: %v", req.String()) - if req == nil { - return nil, grpc.Errorf(codes.InvalidArgument, "nil request") - } - workload, err := convert.ToM3nschWorkload(req.GetWorkload()) - if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "unable to parse workload: %v", err) - } - ms.agent.SetWorkload(workload) - return &proto.ModifyResponse{}, nil -} diff --git a/src/cmd/services/m3nsch_server/tcp/tcp.go b/src/cmd/services/m3nsch_server/tcp/tcp.go deleted file mode 100644 index b5912185af..0000000000 --- a/src/cmd/services/m3nsch_server/tcp/tcp.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package tcp - -import ( - "net" - "time" -) - -// NewTCPListener is Listener specifically for TCP -func NewTCPListener(listenAddress string, keepAlivePeriod time.Duration) (net.Listener, error) { - l, err := net.Listen("tcp", listenAddress) - if err != nil { - return nil, err - } - return tcpKeepAliveListener{l.(*net.TCPListener), keepAlivePeriod}, err -} - -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe and ListenAndServeTLS so -// dead TCP connections (e.g. closing laptop mid-download) eventually -// go away. This is cargo culted from http/server.go -type tcpKeepAliveListener struct { - *net.TCPListener - keepAlivePeriod time.Duration -} - -func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { - tc, err := ln.AcceptTCP() - if err != nil { - return - } - tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(ln.keepAlivePeriod) - return tc, nil -} diff --git a/src/m3nsch/README.md b/src/m3nsch/README.md deleted file mode 100644 index 744aa76f52..0000000000 --- a/src/m3nsch/README.md +++ /dev/null @@ -1,107 +0,0 @@ -m3nsch -====== -m3nsch (pronounced `mensch`) is a load testing tool for M3DB. It has two components: - - `m3nsch_server`: long lived process which does the load generation - - `m3nsch_client`: cli wrapper which controls the myriad `m3nsch_server(s)` - -A typical deploy will have multiple hosts, each running a single `m3nsch_server` instance, -and a single `m3nsch_client` used to control them. - -### Build -``` -$ make prod-m3nsch_server -$ make prod-m3nsch_client -$ cat < server-conf.yaml -server: - listenAddress: "0.0.0.0:12321" - debugAddress: "0.0.0.0:13441" - cpuFactor: 0.9 - -metrics: - sampleRate: 0.1 - m3: - hostPort: "" - service: "m3nsch" - includeHost: true - env: development - -m3nsch: - concurrency: 2000 - numPointsPerDatum: 60 - -# any additional configs you may have -EOF -``` - -### Deploy -(1) Transfer the `m3nsch_server` binary, and `server-conf.yaml` to all the hosts to be used to generate hosts, e.g. Ingesters - -(2) On each host from (1), kick of the server process by running: -``` -./m3nsch_server -f server-conf.yaml -``` - -(3) Transfer `m3nsch_client` binary to a host with network connectivity to all the hosts. - -### Sample Usage -``` -# set an env var containing the host endpoints seperated by commas -$ export ENDPOINTS="host1:12321,host2:12321" - -# get the status of the various endpoints, make sure all are healthy -$ ./m3nsch_client --endpoints $ENDPOINTS status - -# investigate the various options available during initialization -$ ./m3nsch_client init --help -... -Flags: - -b, --basetime-offset duration offset from current time to use for load, e.g. -2m, -30s (default -2m0s) - -c, --cardinality int aggregate workload cardinality (default 10000) - -f, --force force initialization, stop any running workload - -i, --ingress-qps int aggregate workload ingress qps (default 1000) - -p, --metric-prefix string prefix added to each metric (default "m3nsch_") - -n, --namespace string target namespace (default "testmetrics") - -v, --target-env string target env for load test (default "test") - -z, --target-zone string target zone for load test (default "sjc1") - -t, --token string [required] unique identifier required for all subsequent interactions on this workload - -Global Flags: - -e, --endpoints stringSlice host:port for each of the agent process endpoints - -# initialize the servers, set any workload parameters, target env/zone -# the command below targets the production sjc1 m3db cluster with each `m3nsch_server` attempting to -# sustain a load of 100K writes/s from a set of 1M metrics -$ ./m3nsch_client --endpoints $ENDPOINTS init \ - --token prateek-sample \ - --target-env prod \ - --target-zone sjc1 \ - --ingress-qps 100000 \ - --cardinality 1000000 \ - -# start the load generation -$ ./m3nsch_client --endpoints $ENDPOINTS start - -# modifying the running load uses many of the same options as `init` -$ ./m3nsch_client modify --help -... -Flags: - -b, --basetime-offset duration offset from current time to use for load, e.g. -2m, -30s (default -2m0s) - -c, --cardinality int aggregate workload cardinality (default 10000) - -i, --ingress-qps int aggregate workload ingress qps (default 1000) - -p, --metric-prefix string prefix added to each metric (default "m3nsch_") - -n, --namespace string target namespace (default "testmetrics") - -Global Flags: - -e, --endpoints stringSlice host:port for each of the agent process endpoints - -# the command below bumps up the workload on each `m3nsch_server`, to sustain -# a load of 1M writes/s from a set of 10M metrics -$ ./m3nsch_client --endpoints $ENDPOINTS - --ingress-qps 1000000 \ - --cardinality 10000000 \ - -# finally, stop the load -$ ./m3nsch_client --endpoints $ENDPOINTS stop - -# probably want to teardown the running server processes on the various hosts -``` diff --git a/src/m3nsch/agent/agent.go b/src/m3nsch/agent/agent.go deleted file mode 100644 index 99216b6eb8..0000000000 --- a/src/m3nsch/agent/agent.go +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package agent - -import ( - "errors" - "fmt" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/datums" - "github.com/m3db/m3/src/x/ident" - "github.com/m3db/m3/src/x/instrument" - xtime "github.com/m3db/m3/src/x/time" - - "go.uber.org/zap" -) - -var ( - errCannotStartNotInitialized = errors.New("unable to start, agent is not initialized") - errCannotStopNotInitialized = errors.New("unable to stop, agent is not initialized") - errAlreadyInitialized = errors.New("unable to initialize, agent already initialized") -) - -type m3nschAgent struct { - sync.RWMutex - token string // workload token - workload m3nsch.Workload // workload to operate upon - registry datums.Registry // workload fake metric registry - session client.Session // m3db session to operate upon - agentStatus m3nsch.Status // agent status - opts m3nsch.AgentOptions // agent options - logger *zap.Logger // logger - metrics agentMetrics // agent performance metrics - workerChans workerChannels // worker-idx -> channel for worker notification - workerWg sync.WaitGroup // used to track when workers are finished - params workerParams // worker params - lastStartTime int64 // last time a workload was started as unix epoch -} - -type workerParams struct { - sync.RWMutex - fn workerFn // workerFn (read|write) - workingSet []generatedMetric // metrics corresponding to workload - ranges []workerRange // worker-idx -> workingSet idx range -} - -// New returns a new Agent. -func New( - registry datums.Registry, - opts m3nsch.AgentOptions, -) m3nsch.Agent { - ms := &m3nschAgent{ - registry: registry, - opts: opts, - logger: opts.InstrumentOptions().Logger(), - params: workerParams{ - fn: workerWriteFn, - }, - } - ms.metrics = agentMetrics{ - writeMethodMetrics: ms.newMethodMetrics("write"), - } - return ms - -} - -func newWorkerChannels(numWorkers int) workerChannels { - var chans workerChannels - for i := 0; i < numWorkers; i++ { - chans = append(chans, make(chan workerNotification)) - } - return chans -} - -func (ms *m3nschAgent) closeWorkerChannelsWithLock() { - ms.workerChans.close() - ms.workerChans = nil -} - -func (ms *m3nschAgent) notifyWorkersWithLock(msg workerNotification) { - ms.workerChans.notify(msg) -} - -func (ms *m3nschAgent) resetWithLock() { - if ms.workerChans != nil { - ms.closeWorkerChannelsWithLock() - } - if ms.session != nil { - ms.session.Close() - ms.session = nil - } - ms.workload = m3nsch.Workload{} - ms.agentStatus = m3nsch.StatusUninitialized - ms.params.workingSet = nil - ms.params.ranges = nil -} - -func (ms *m3nschAgent) setWorkerParams(workload m3nsch.Workload) { - ms.params.Lock() - defer ms.params.Unlock() - - var ( - current = ms.params.workingSet - cardinality = workload.Cardinality - ) - - if len(current) > cardinality { - current = current[:cardinality] - } - for i := len(current); i < cardinality; i++ { - idx := workload.MetricStartIdx + i - current = append(current, generatedMetric{ - name: fmt.Sprintf("%v.m%d", workload.MetricPrefix, idx), - timeseries: ms.registry.Get(i), - }) - } - ms.params.workingSet = current - - concurrency := ms.opts.Concurrency() - numMetricsPerWorker := len(current) / concurrency - workerRanges := make([]workerRange, concurrency) - for i := 0; i < concurrency; i++ { - workerRanges[i] = workerRange{ - startIdx: i * numMetricsPerWorker, - endIdx: (i+1)*numMetricsPerWorker - 1, - lastIdx: -1, - } - } - ms.params.ranges = workerRanges -} - -func (ms *m3nschAgent) Status() m3nsch.AgentStatus { - ms.RLock() - defer ms.RUnlock() - return m3nsch.AgentStatus{ - Status: ms.agentStatus, - Token: ms.token, - } -} - -func (ms *m3nschAgent) Workload() m3nsch.Workload { - ms.RLock() - defer ms.RUnlock() - return ms.workload -} - -func (ms *m3nschAgent) SetWorkload(w m3nsch.Workload) { - ms.Lock() - defer ms.Unlock() - ms.workload = w - ms.setWorkerParams(w) - - if ms.agentStatus == m3nsch.StatusRunning { - ms.notifyWorkersWithLock(workerNotification{update: true}) - } -} - -func (ms *m3nschAgent) Init( - token string, - w m3nsch.Workload, - force bool, - targetZone string, - targetEnv string, -) error { - ms.Lock() - defer ms.Unlock() - status := ms.agentStatus - if status != m3nsch.StatusUninitialized && !force { - return errAlreadyInitialized - } - - if status == m3nsch.StatusRunning { - if err := ms.stopWithLock(); err != nil { - return err - } - } - - session, err := ms.opts.NewSessionFn()(targetZone, targetEnv) - if err != nil { - return err - } - ms.session = session - ms.token = token - ms.workload = w - ms.setWorkerParams(w) - ms.agentStatus = m3nsch.StatusInitialized - return nil -} - -func (ms *m3nschAgent) Start() error { - ms.Lock() - defer ms.Unlock() - if ms.agentStatus != m3nsch.StatusInitialized { - return errCannotStartNotInitialized - } - concurrency := ms.opts.Concurrency() - ms.workerChans = newWorkerChannels(concurrency) - ms.agentStatus = m3nsch.StatusRunning - atomic.StoreInt64(&ms.lastStartTime, time.Now().Unix()) - ms.workerWg.Add(concurrency) - for i := 0; i < concurrency; i++ { - go ms.runWorker(i, ms.workerChans[i]) - } - return nil -} - -func (ms *m3nschAgent) stopWithLock() error { - status := ms.agentStatus - if status == m3nsch.StatusUninitialized { - return errCannotStopNotInitialized - } - - if status == m3nsch.StatusRunning { - ms.notifyWorkersWithLock(workerNotification{stop: true}) - ms.workerWg.Wait() - } - - ms.resetWithLock() - return nil -} - -func (ms *m3nschAgent) Stop() error { - ms.Lock() - defer ms.Unlock() - return ms.stopWithLock() -} - -func (ms *m3nschAgent) MaxQPS() int64 { - return int64(ms.opts.Concurrency()) * ms.opts.MaxWorkerQPS() -} - -// nolint: unparam -func (ms *m3nschAgent) newMethodMetrics(method string) instrument.MethodMetrics { - subScope := ms.opts.InstrumentOptions().MetricsScope().SubScope("agent") - return instrument.NewMethodMetrics(subScope, method, ms.opts.InstrumentOptions().TimerOptions()) -} - -func (ms *m3nschAgent) tickPeriodWithLock() time.Duration { - var ( - qps = ms.workload.IngressQPS - numWorkers = ms.opts.Concurrency() - qpsPerWorker = float64(qps) / float64(numWorkers) - tickPeriod = time.Duration(1000*1000*1000/qpsPerWorker) * time.Nanosecond - ) - return tickPeriod -} - -func (ms *m3nschAgent) workerParams() (xtime.Unit, string, time.Time, time.Duration) { - ms.params.RLock() - defer ms.params.RUnlock() - return ms.opts.TimeUnit(), ms.workload.Namespace, ms.workload.BaseTime, ms.tickPeriodWithLock() -} - -func (ms *m3nschAgent) nextWorkerMetric(workerIdx int) generatedMetric { - ms.params.RLock() - defer ms.params.RUnlock() - metricIdx := ms.params.ranges[workerIdx].next() - return ms.params.workingSet[metricIdx] -} - -func (ms *m3nschAgent) runWorker(workerIdx int, workerCh chan workerNotification) { - defer ms.workerWg.Done() - var ( - methodMetrics = ms.metrics.writeMethodMetrics - timeUnit, namespace, fakeNow, tickPeriod = ms.workerParams() - tickLoop = time.NewTicker(tickPeriod) - ) - defer tickLoop.Stop() - for { - select { - case msg := <-workerCh: - if msg.stop { - return - } - if msg.update { - tickLoop.Stop() - timeUnit, namespace, fakeNow, tickPeriod = ms.workerParams() - tickLoop = time.NewTicker(tickPeriod) - } - - case <-tickLoop.C: - fakeNow = fakeNow.Add(tickPeriod) - metric := ms.nextWorkerMetric(workerIdx) - start := time.Now() - - // If configured to generate uniques over time, modify the metric to add - // cardinality. - if u := ms.workload.UniqueAmplifier; u > 0 { - lastStart := time.Unix(atomic.LoadInt64(&ms.lastStartTime), 0) - suffix := "/" + metricUniqueSuffix(lastStart, start, u) - metric.name += suffix - } - - err := ms.params.fn(workerIdx, ms.session, namespace, metric, fakeNow, timeUnit) - elapsed := time.Since(start) - methodMetrics.ReportSuccessOrError(err, elapsed) - } - } -} - -func metricUniqueSuffix(startTime, now time.Time, uniqueAmplifier float64) string { - n := now.Sub(startTime).Seconds() * uniqueAmplifier - return strconv.Itoa(int(n)) -} - -type generatedMetric struct { - name string - timeseries datums.SyntheticTimeSeries -} - -type workerRange struct { - startIdx int // inclusive - endIdx int // exclusive - lastIdx int // last idx returned -} - -func (wr *workerRange) next() int { - i := wr.lastIdx - next := wr.startIdx + ((i + 1) % (wr.endIdx - wr.startIdx + 1)) - wr.lastIdx = next - return next -} - -type workerNotification struct { - stop bool - update bool -} - -type workerChannels []chan workerNotification - -func (c workerChannels) close() { - for _, ch := range c { - close(ch) - } -} - -func (c workerChannels) notify(msg workerNotification) { - for _, ch := range c { - ch <- msg - } -} - -type agentMetrics struct { - writeMethodMetrics instrument.MethodMetrics -} - -type workerFn func(workerIdx int, session client.Session, namespace string, metric generatedMetric, t time.Time, u xtime.Unit) error - -func workerWriteFn(_ int, session client.Session, namespace string, metric generatedMetric, t time.Time, u xtime.Unit) error { - return session.Write(ident.StringID(namespace), ident.StringID(metric.name), t, metric.timeseries.Next(), u, nil) -} diff --git a/src/m3nsch/agent/agent_options.go b/src/m3nsch/agent/agent_options.go deleted file mode 100644 index fb5aef9a91..0000000000 --- a/src/m3nsch/agent/agent_options.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package agent - -import ( - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/x/instrument" - xtime "github.com/m3db/m3/src/x/time" -) - -var ( - // avg latency per m3db write op is ~1 ms when the CPU is under load - // so liberally, we set MaxWorkerQPS at ~ 10K writes per sec - defaultMaxWorkerQPS = int64(10000) - - // defaultConcurrency is the default number of go routines used during - // load generation - defaultConcurrency = int(2000) - - // defaultTimeUnit is the default unit of time used during load operations - defaultTimeUnit = xtime.Second -) - -type agentOpts struct { - iopts instrument.Options - maxWorkerQPS int64 - concurrency int - newSessionFn m3nsch.NewSessionFn - timeUnit xtime.Unit -} - -// NewOptions returns a new AgentOptions object with default values -func NewOptions( - iopts instrument.Options, -) m3nsch.AgentOptions { - return &agentOpts{ - iopts: iopts, - maxWorkerQPS: defaultMaxWorkerQPS, - concurrency: defaultConcurrency, - timeUnit: defaultTimeUnit, - } -} - -func (so *agentOpts) SetInstrumentOptions(iopts instrument.Options) m3nsch.AgentOptions { - so.iopts = iopts - return so -} - -func (so *agentOpts) InstrumentOptions() instrument.Options { - return so.iopts -} - -func (so *agentOpts) SetMaxWorkerQPS(qps int64) m3nsch.AgentOptions { - so.maxWorkerQPS = qps - return so -} - -func (so *agentOpts) MaxWorkerQPS() int64 { - return so.maxWorkerQPS -} - -func (so *agentOpts) SetConcurrency(c int) m3nsch.AgentOptions { - so.concurrency = c - return so -} - -func (so *agentOpts) Concurrency() int { - return so.concurrency -} - -func (so *agentOpts) SetNewSessionFn(fn m3nsch.NewSessionFn) m3nsch.AgentOptions { - so.newSessionFn = fn - return so -} - -func (so *agentOpts) NewSessionFn() m3nsch.NewSessionFn { - return so.newSessionFn -} - -func (so *agentOpts) SetTimeUnit(u xtime.Unit) m3nsch.AgentOptions { - so.timeUnit = u - return so -} - -func (so *agentOpts) TimeUnit() xtime.Unit { - return so.timeUnit -} diff --git a/src/m3nsch/agent/agent_test.go b/src/m3nsch/agent/agent_test.go deleted file mode 100644 index 36f8d38c20..0000000000 --- a/src/m3nsch/agent/agent_test.go +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package agent - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/datums" - "github.com/m3db/m3/src/x/instrument" - xtime "github.com/m3db/m3/src/x/time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - testNumPointsPerDatum = 40 -) - -type testWrite struct { - metricName string - timestamp time.Time - value float64 -} - -func newTestOptions() m3nsch.AgentOptions { - iopts := instrument.NewOptions() - return NewOptions(iopts). - SetNewSessionFn(func(_, _ string) (client.Session, error) { - return nil, nil - }) -} - -func TestWorkloadMetricStartIdx(t *testing.T) { - var ( - reg = datums.NewDefaultRegistry(testNumPointsPerDatum) - opts = newTestOptions(). - SetConcurrency(1) - workload = m3nsch.Workload{ - Cardinality: 10, - IngressQPS: 100, - MetricStartIdx: 1000, - } - agent = New(reg, opts).(*m3nschAgent) - writes []testWrite - - token = "" - targetZone = "" - targetEnv = "" - ) - - agent.params.fn = func(_ int, _ client.Session, _ string, metric generatedMetric, t time.Time, _ xtime.Unit) error { - writes = append(writes, testWrite{ - metricName: metric.name, - timestamp: t, - value: metric.timeseries.Next(), - }) - return nil - } - - err := agent.Init(token, workload, false, targetZone, targetEnv) - require.NoError(t, err) - - err = agent.Start() - require.NoError(t, err) - - // let worker perform write ops for 1 second - time.Sleep(1 * time.Second) - - err = agent.Stop() - require.NoError(t, err) - - // ensure we've seen 90% of the writes we're expecting - eps := 0.1 - numExpectedWrites := workload.IngressQPS - require.InEpsilon(t, numExpectedWrites, len(writes), eps) - - // ensure the ordering of metric writes is correct - for i, wr := range writes { - metricIdx := workload.MetricStartIdx + i%workload.Cardinality - require.Equal(t, fmt.Sprintf(".m%d", metricIdx), wr.metricName) - } -} - -func TestNewSingleWriterAgent(t *testing.T) { - var ( - reg = datums.NewDefaultRegistry(testNumPointsPerDatum) - opts = newTestOptions(). - SetConcurrency(1) - workload = m3nsch.Workload{ - Cardinality: 10, - IngressQPS: 100, - } - agent = New(reg, opts).(*m3nschAgent) - writes []testWrite - - token = "" - targetZone = "" - targetEnv = "" - ) - - agent.params.fn = func(_ int, _ client.Session, _ string, metric generatedMetric, t time.Time, _ xtime.Unit) error { - writes = append(writes, testWrite{ - metricName: metric.name, - timestamp: t, - value: metric.timeseries.Next(), - }) - return nil - } - - err := agent.Init(token, workload, false, targetZone, targetEnv) - require.NoError(t, err) - - err = agent.Start() - require.NoError(t, err) - - // let worker perform write ops for 1 second - time.Sleep(1 * time.Second) - - err = agent.Stop() - require.NoError(t, err) - - // ensure we've seen 90% of the writes we're expecting - eps := 0.1 - numExpectedWrites := workload.IngressQPS - require.InEpsilon(t, numExpectedWrites, len(writes), eps) - - // ensure the ordering of metric writes is correct - for i, wr := range writes { - metricIdx := i % workload.Cardinality - require.Equal(t, fmt.Sprintf(".m%d", metricIdx), wr.metricName) - } - - // ensure the values written per metric are accurate - // first, group all values by metricIdx (which are the same as metricName due to assertion above) - valuesByMetricIdx := make(map[int][]testWrite) - for i, wr := range writes { - metricIdx := i % workload.Cardinality - current, ok := valuesByMetricIdx[metricIdx] - if !ok { - current = []testWrite{} - } - current = append(current, wr) - valuesByMetricIdx[metricIdx] = current - } - - // finally, go through the values per metric, and ensure - // they line up with expected values from registry - for idx, values := range valuesByMetricIdx { - datum := reg.Get(idx) - for i, wr := range values { - require.Equal(t, datum.Get(i), wr.value, - "metric: %s, idx: %d, i: %d, ts: %s", wr.metricName, idx, i, wr.timestamp.String()) - } - } -} - -func TestWorkerParams(t *testing.T) { - var ( - reg = datums.NewDefaultRegistry(testNumPointsPerDatum) - opts = newTestOptions(). - SetConcurrency(10) - agent = New(reg, opts).(*m3nschAgent) - t0 = time.Now() - testNs = "testNs" - workload = m3nsch.Workload{ - Cardinality: 1000, - IngressQPS: 100, - BaseTime: t0, - Namespace: testNs, - } - ) - agent.SetWorkload(workload) - - expectedWritesPerWorkerPerSec := workload.IngressQPS / opts.Concurrency() - expectedTickPeriodPerWorker := time.Duration(1000.0/float64(expectedWritesPerWorkerPerSec)) * time.Millisecond - - _, ns, baseTime, tickPeriod := agent.workerParams() - require.Equal(t, testNs, ns) - require.Equal(t, t0, baseTime) - require.Equal(t, expectedTickPeriodPerWorker, tickPeriod) -} - -func TestMultipleWriterAgent(t *testing.T) { - var ( - reg = datums.NewDefaultRegistry(testNumPointsPerDatum) - opts = newTestOptions(). - SetConcurrency(2) - workload = m3nsch.Workload{ - Cardinality: 4, - IngressQPS: 100, - } - token = "" - targetZone = "" - targetEnv = "" - agent = New(reg, opts).(*m3nschAgent) - - writesLock sync.Mutex - writesByWorkerIdx map[int][]testWrite - ) - - // initialize writesByWorkerIdx - writesByWorkerIdx = make(map[int][]testWrite) - for i := 0; i < opts.Concurrency(); i++ { - writesByWorkerIdx[i] = []testWrite{} - } - - agent.params.fn = func(wIdx int, _ client.Session, _ string, metric generatedMetric, t time.Time, _ xtime.Unit) error { - writesLock.Lock() - writesByWorkerIdx[wIdx] = append(writesByWorkerIdx[wIdx], testWrite{ - metricName: metric.name, - timestamp: t, - value: metric.timeseries.Next(), - }) - writesLock.Unlock() - return nil - } - - err := agent.Init(token, workload, false, targetZone, targetEnv) - require.NoError(t, err) - - err = agent.Start() - require.NoError(t, err) - - // let worker perform write ops for 1 second - time.Sleep(1 * time.Second) - - err = agent.Stop() - require.NoError(t, err) - - // ensure we've seen at least 10% of the expected writes - // NB(prateek): ideally, this would require a stricter number of writes - // but in testing, the mutex overhead induced due to protecting the - // observed writes map is large. and for the purposes of testing, - // 10% of the values are sufficient for testing ordering assertions - eps := 0.10 - numExpectedWritesPerWorker := int(float64(workload.IngressQPS) * eps) - for i := 0; i < opts.Concurrency(); i++ { - numSeenWrites := len(writesByWorkerIdx[i]) - require.True(t, numSeenWrites > numExpectedWritesPerWorker, - "worker: %d, expectedWrites: %d, seenWrites: %d", i) - } - - // helper to identify which metric we're expecting per worker - metricIdxFn := func(wIdx int, mIdx int) int { - numWorkers := opts.Concurrency() - numMetrics := workload.Cardinality - numMetricsPerWorker := numMetrics / numWorkers - idx := numMetricsPerWorker*wIdx + mIdx%numMetricsPerWorker - return idx - } - - // ensure the ordering of metric writes is correct - for workerIdx, writes := range writesByWorkerIdx { - for i, wr := range writes { - metricIdx := metricIdxFn(workerIdx, i) - require.Equal(t, fmt.Sprintf(".m%d", metricIdx), wr.metricName) - } - } -} - -// test for transition checks -func TestTransitions(t *testing.T) { - var ( - reg = datums.NewDefaultRegistry(testNumPointsPerDatum) - opts = newTestOptions(). - SetConcurrency(10) - agent = New(reg, opts).(*m3nschAgent) - workload = m3nsch.Workload{ - Cardinality: 1000, - IngressQPS: 100, - } - ) - agent.params.fn = func(_ int, _ client.Session, _ string, _ generatedMetric, _ time.Time, _ xtime.Unit) error { - return nil - } - - err := agent.Start() - require.Error(t, err) - - err = agent.Stop() - require.Error(t, err) - - err = agent.Init("", workload, false, "", "") - require.NoError(t, err) - - err = agent.Stop() - require.NoError(t, err) - - err = agent.Init("", workload, false, "", "") - require.NoError(t, err) - - err = agent.Start() - require.NoError(t, err) - - err = agent.Stop() - require.NoError(t, err) -} - -func TestMetricUniqueSuffix(t *testing.T) { - start := time.Unix(0, 0) - for u, exp := range map[float64]int{ - 0: 1, - 0.2: 2, - 0.5: 5, - 1.0: 10, - } { - uniques := make(map[string]struct{}) - for i := 0; i < 10; i++ { - now := start.Add(time.Duration(i) * time.Second) - n := metricUniqueSuffix(start, now, u) - uniques[n] = struct{}{} - } - l := len(uniques) - assert.Equal(t, exp, l, "expected to see %d unique metrics", l) - } -} diff --git a/src/m3nsch/coordinator/coordinator.go b/src/m3nsch/coordinator/coordinator.go deleted file mode 100644 index 2a0899eacb..0000000000 --- a/src/m3nsch/coordinator/coordinator.go +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package coordinator - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/generated/convert" - proto "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch" - xerrors "github.com/m3db/m3/src/x/errors" - "github.com/m3db/m3/src/x/instrument" - - "go.uber.org/zap" - "google.golang.org/grpc" -) - -var ( - errNoEndpointsProvided = fmt.Errorf("no endpoints provided") -) - -type m3nschCoordinator struct { - sync.Mutex - opts m3nsch.CoordinatorOptions - clients map[string]*m3nschClient - newClientFn newM3nschClientFn -} - -// New returns a new Coordinator process with provided endpoints treated -// as Agent processes. -func New( - opts m3nsch.CoordinatorOptions, - endpoints []string, -) (m3nsch.Coordinator, error) { - if len(endpoints) == 0 { - return nil, errNoEndpointsProvided - } - coordinator := &m3nschCoordinator{ - opts: opts, - clients: make(map[string]*m3nschClient), - newClientFn: newM3nschClient, - } - return coordinator, coordinator.initializeConnections(endpoints) -} - -func (m *m3nschCoordinator) initializeConnections(endpoints []string) error { - var ( - wg sync.WaitGroup - numEndpoints = len(endpoints) - multiErr syncClientMultiErr - ) - - wg.Add(numEndpoints) - for i := 0; i < numEndpoints; i++ { - go func(idx int) { - defer wg.Done() - endpoint := endpoints[idx] - client, err := m.newClientFn(endpoint, m.opts.InstrumentOptions(), m.opts.Timeout()) - if err != nil { - multiErr.Add(endpoint, err) - return - } - m.Lock() - m.clients[endpoint] = client - m.Unlock() - }(i) - } - wg.Wait() - - return multiErr.FinalError() -} - -func (m *m3nschCoordinator) Teardown() error { - var err syncClientMultiErr - m.forEachClient(func(c *m3nschClient) { - err.Add(c.endpoint, c.close()) - }) - return err.FinalError() -} - -func (m *m3nschCoordinator) Status() (map[string]m3nsch.AgentStatus, error) { - var ( - ctx = context.Background() - multiErr syncClientMultiErr - - lock sync.Mutex - statuses = make(map[string]m3nsch.AgentStatus, len(m.clients)) - ) - - m.forEachClient(func(c *m3nschClient) { - response, err := c.client.Status(ctx, &proto.StatusRequest{}) - if err != nil { - multiErr.Add(c.endpoint, err) - return - } - - status, err := convert.ToM3nschStatus(response.Status) - if err != nil { - multiErr.Add(c.endpoint, err) - return - } - - workload, err := convert.ToM3nschWorkload(response.GetWorkload()) - if err != nil { - multiErr.Add(c.endpoint, err) - return - } - - lock.Lock() - statuses[c.endpoint] = m3nsch.AgentStatus{ - Status: status, - Token: response.Token, - MaxQPS: response.MaxQPS, - Workload: workload, - } - lock.Unlock() - }) - - return statuses, nil -} - -func (m *m3nschCoordinator) Workload() (m3nsch.Workload, error) { - statuses, err := m.Status() - if err != nil { - return m3nsch.Workload{}, err - } - - // ensure all agents are initialized - var multiErr syncClientMultiErr - for endpoint, status := range statuses { - if status.Status == m3nsch.StatusUninitialized { - multiErr.Add(endpoint, fmt.Errorf("agent not initialized")) - } - } - if err = multiErr.FinalError(); err != nil { - return m3nsch.Workload{}, err - } - - // TODO(prateek): ensure all agent workloads are set the same - // TODO(prateek): ensure no agent have overlapping metric ranges - var ( - workload = m3nsch.Workload{} - first = true - ) - for _, status := range statuses { - if first { - workload.BaseTime = status.Workload.BaseTime - workload.Namespace = status.Workload.Namespace - workload.MetricPrefix = status.Workload.MetricPrefix - first = false - } - workload.Cardinality += status.Workload.Cardinality - workload.IngressQPS += status.Workload.IngressQPS - } - - return workload, nil -} - -func (m *m3nschCoordinator) SetWorkload(workload m3nsch.Workload) error { - statuses, err := m.Status() - if err != nil { - return err - } - - // ensure all agents are initialized - var multiErr syncClientMultiErr - for endpoint, status := range statuses { - if status.Status == m3nsch.StatusUninitialized { - multiErr.Add(endpoint, fmt.Errorf("agent not initialized")) - } - } - if err = multiErr.FinalError(); err != nil { - return err - } - - // split workload amongst the various agents - splitWorkload, err := m.splitWorkload(workload, statuses) - if err != nil { - return err - } - - // set the targetWorkload on each agent - ctx := context.Background() - multiErr = syncClientMultiErr{} - m.forEachClient(func(c *m3nschClient) { - endpoint := c.endpoint - targetWorkload, ok := splitWorkload[endpoint] - if !ok { - multiErr.Add(endpoint, fmt.Errorf("splitWorkload does not contain all endpoints")) - return - } - - rpcWorkload := convert.ToProtoWorkload(targetWorkload) - _, clientErr := c.client.Modify(ctx, &proto.ModifyRequest{Workload: &rpcWorkload}) - multiErr.Add(endpoint, clientErr) - }) - - return multiErr.FinalError() -} - -func (m *m3nschCoordinator) Init( - token string, - workload m3nsch.Workload, - force bool, - targetZone string, - targetEnv string, -) error { - statuses, err := m.Status() - if err != nil { - return err - } - - // ensure all agents are not initialized - var multiErr syncClientMultiErr - for endpoint, status := range statuses { - if status.Status != m3nsch.StatusUninitialized { - multiErr.Add(endpoint, fmt.Errorf("agent already initialized")) - } - } - if err = multiErr.FinalError(); err != nil { - return err - } - - // split workload amongst the various agents - splitWorkload, err := m.splitWorkload(workload, statuses) - if err != nil { - return err - } - - // initialize each agent with targetWorkload - ctx := context.Background() - multiErr = syncClientMultiErr{} - m.forEachClient(func(c *m3nschClient) { - endpoint := c.endpoint - targetWorkload, ok := splitWorkload[endpoint] - if !ok { - multiErr.Add(endpoint, fmt.Errorf("splitWorkload does not contain all endpoints")) - return - } - - rpcWorkload := convert.ToProtoWorkload(targetWorkload) - _, clientErr := c.client.Init(ctx, &proto.InitRequest{ - Token: token, - Workload: &rpcWorkload, - Force: force, - TargetZone: targetZone, - TargetEnv: targetEnv, - }) - multiErr.Add(endpoint, clientErr) - }) - - return multiErr.FinalError() -} - -func (m *m3nschCoordinator) Start() error { - statuses, err := m.Status() - if err != nil { - return err - } - - // ensure all agents are initialized - var multiErr syncClientMultiErr - for endpoint, status := range statuses { - if status.Status == m3nsch.StatusUninitialized { - multiErr.Add(endpoint, fmt.Errorf("agent not initialized")) - } - } - if err = multiErr.FinalError(); err != nil { - return err - } - - // start each agent - ctx := context.Background() - multiErr = syncClientMultiErr{} - m.forEachClient(func(c *m3nschClient) { - endpoint := c.endpoint - _, clientErr := c.client.Start(ctx, &proto.StartRequest{}) - multiErr.Add(endpoint, clientErr) - }) - - return multiErr.FinalError() -} - -func (m *m3nschCoordinator) Stop() error { - // stop each agent - ctx := context.Background() - multiErr := syncClientMultiErr{} - m.forEachClient(func(c *m3nschClient) { - endpoint := c.endpoint - _, err := c.client.Stop(ctx, &proto.StopRequest{}) - multiErr.Add(endpoint, err) - }) - - return multiErr.FinalError() -} - -func (m *m3nschCoordinator) splitWorkload( - aggWorkload m3nsch.Workload, - statuses map[string]m3nsch.AgentStatus, -) (map[string]m3nsch.Workload, error) { - // ensure we have enough aggregate capacity to satisfy workload - totalIngressCapacity := int64(0) - for _, status := range statuses { - totalIngressCapacity += status.MaxQPS - } - if totalIngressCapacity < int64(aggWorkload.IngressQPS) { - return nil, fmt.Errorf("insufficient capacity") - } - - // initialiaze return map - splitWorkload := make(map[string]m3nsch.Workload, len(statuses)) - - // split workload per worker proportional to capacity - metricStart := 0 - for endpoint, status := range statuses { - var ( - workerFrac = float64(status.MaxQPS) / float64(totalIngressCapacity) - numMetrics = int(float64(aggWorkload.Cardinality) * workerFrac) - qps = int(float64(aggWorkload.IngressQPS) * workerFrac) - workerWorkload = aggWorkload - ) - workerWorkload.MetricStartIdx = metricStart - workerWorkload.Cardinality = numMetrics - workerWorkload.IngressQPS = qps - splitWorkload[endpoint] = workerWorkload - - metricStart += numMetrics - } - return splitWorkload, nil -} - -type syncClientMultiErr struct { - sync.Mutex - multiErr xerrors.MultiError -} - -func (e *syncClientMultiErr) Add(endpoint string, err error) { - if err == nil { - return - } - cErr := fmt.Errorf("[%v] %v", endpoint, err) - e.Lock() - e.multiErr = e.multiErr.Add(cErr) - e.Unlock() -} - -func (e *syncClientMultiErr) FinalError() error { - return e.multiErr.FinalError() -} - -type forEachClientFn func(client *m3nschClient) - -func (m *m3nschCoordinator) forEachClient(fn forEachClientFn) { - var ( - parallel = m.opts.ParallelOperations() - numClients = len(m.clients) - wg sync.WaitGroup - ) - - if parallel { - wg.Add(numClients) - } - - for _, client := range m.clients { - if parallel { - go func(c *m3nschClient) { - defer wg.Done() - fn(c) - }(client) - } else { - fn(client) - } - } - - if parallel { - wg.Wait() - } -} - -type newM3nschClientFn func(string, instrument.Options, time.Duration) (*m3nschClient, error) - -type m3nschClient struct { - endpoint string - logger *zap.Logger - conn *grpc.ClientConn - client proto.MenschClient -} - -func newM3nschClient( - endpoint string, - iopts instrument.Options, - timeout time.Duration, -) (*m3nschClient, error) { - var ( - logger = iopts.Logger().With(zap.String("client", endpoint)) - conn, err = grpc.Dial(endpoint, grpc.WithTimeout(timeout), grpc.WithInsecure()) - ) - if err != nil { - return nil, fmt.Errorf("could not connect: %v", err) - } - logger.Debug("connection established") - client := proto.NewMenschClient(conn) - return &m3nschClient{ - endpoint: endpoint, - logger: logger, - conn: conn, - client: client, - }, nil -} - -func (mc *m3nschClient) close() error { - return mc.conn.Close() -} diff --git a/src/m3nsch/coordinator/coordinator_options.go b/src/m3nsch/coordinator/coordinator_options.go deleted file mode 100644 index df5a3ed42d..0000000000 --- a/src/m3nsch/coordinator/coordinator_options.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package coordinator - -import ( - "time" - - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/x/instrument" -) - -var ( - defaultRPCTimeout = time.Minute - defaultParallelOps = true -) - -type coordinatorOpts struct { - iopts instrument.Options - timeout time.Duration - parallel bool -} - -// NewOptions creates a new options struct. -func NewOptions( - iopts instrument.Options, -) m3nsch.CoordinatorOptions { - return &coordinatorOpts{ - iopts: iopts, - timeout: defaultRPCTimeout, - parallel: defaultParallelOps, - } -} - -func (mo *coordinatorOpts) SetInstrumentOptions(iopts instrument.Options) m3nsch.CoordinatorOptions { - mo.iopts = iopts - return mo -} - -func (mo *coordinatorOpts) InstrumentOptions() instrument.Options { - return mo.iopts -} - -func (mo *coordinatorOpts) SetTimeout(d time.Duration) m3nsch.CoordinatorOptions { - mo.timeout = d - return mo -} - -func (mo *coordinatorOpts) Timeout() time.Duration { - return mo.timeout -} - -func (mo *coordinatorOpts) SetParallelOperations(f bool) m3nsch.CoordinatorOptions { - mo.parallel = f - return mo -} - -func (mo *coordinatorOpts) ParallelOperations() bool { - return mo.parallel -} diff --git a/src/m3nsch/coordinator/coordinator_test.go b/src/m3nsch/coordinator/coordinator_test.go deleted file mode 100644 index a00ad7f96a..0000000000 --- a/src/m3nsch/coordinator/coordinator_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package coordinator - -import ( - "sync" - "testing" - "time" - - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/x/instrument" - - "github.com/stretchr/testify/require" -) - -var ( - testEndpoints = []string{ - "testEndpoint1", - "testEndpoint2", - } -) - -func newTestCoordinator() *m3nschCoordinator { - iopts := instrument.NewOptions() - opts := NewOptions(iopts) - return &m3nschCoordinator{ - opts: opts, - clients: make(map[string]*m3nschClient), - } -} - -func TestInitializeConnections(t *testing.T) { - var ( - lock sync.Mutex - initMap = make(map[string]bool) - coordinator = newTestCoordinator() - ) - coordinator.newClientFn = func(e string, _ instrument.Options, _ time.Duration) (*m3nschClient, error) { - lock.Lock() - initMap[e] = true - lock.Unlock() - return &m3nschClient{}, nil - } - - err := coordinator.initializeConnections(testEndpoints) - require.NoError(t, err) - - for _, endpoint := range testEndpoints { - flag, ok := initMap[endpoint] - require.True(t, ok, "endpoint not initialized: %v", endpoint) - require.True(t, flag, "endpoint not initialized: %v", endpoint) - } -} - -func TestForEachClient(t *testing.T) { - coordinator := newTestCoordinator() - coordinator.newClientFn = func(e string, _ instrument.Options, _ time.Duration) (*m3nschClient, error) { - return &m3nschClient{endpoint: e}, nil - } - err := coordinator.initializeConnections(testEndpoints) - require.NoError(t, err) - - // non-parallel version - coordinator.opts = coordinator.opts.SetParallelOperations(false) - clientMap := make(map[string]bool) - coordinator.forEachClient(func(c *m3nschClient) { - clientMap[c.endpoint] = true - }) - for _, endpoint := range testEndpoints { - flag, ok := clientMap[endpoint] - require.True(t, ok, "endpoint not initialized: %v", endpoint) - require.True(t, flag, "endpoint not initialized: %v", endpoint) - } - - // parallel version - var lock sync.Mutex - coordinator.opts = coordinator.opts.SetParallelOperations(true) - clientMap = make(map[string]bool) - coordinator.forEachClient(func(c *m3nschClient) { - lock.Lock() - clientMap[c.endpoint] = true - lock.Unlock() - }) - for _, endpoint := range testEndpoints { - flag, ok := clientMap[endpoint] - require.True(t, ok, "endpoint not initialized: %v", endpoint) - require.True(t, flag, "endpoint not initialized: %v", endpoint) - } -} - -func TestSplitWorkloadFail(t *testing.T) { - coordinator := newTestCoordinator() - aggregateWorkload := m3nsch.Workload{ - IngressQPS: 2, - } - statuses := map[string]m3nsch.AgentStatus{ - testEndpoints[0]: { - MaxQPS: 1, - }, - } - _, err := coordinator.splitWorkload(aggregateWorkload, statuses) - require.Error(t, err) -} - -func TestSplitWorkload(t *testing.T) { - coordinator := newTestCoordinator() - aggregateWorkload := m3nsch.Workload{ - Cardinality: 3000, - IngressQPS: 300, - } - statuses := map[string]m3nsch.AgentStatus{ - testEndpoints[0]: { - MaxQPS: 200, - }, - testEndpoints[1]: { - MaxQPS: 400, - }, - } - splitWorkloads, err := coordinator.splitWorkload(aggregateWorkload, statuses) - require.NoError(t, err) - require.Equal(t, 2, len(splitWorkloads)) - - workload1, ok := splitWorkloads[testEndpoints[0]] - require.True(t, ok) - require.Equal(t, 1000, workload1.Cardinality) - require.Equal(t, 100, workload1.IngressQPS) - - workload2, ok := splitWorkloads[testEndpoints[1]] - require.True(t, ok) - require.Equal(t, 2000, workload2.Cardinality) - require.Equal(t, 200, workload2.IngressQPS) -} diff --git a/src/m3nsch/datums/datum.go b/src/m3nsch/datums/datum.go deleted file mode 100644 index 694d182a0e..0000000000 --- a/src/m3nsch/datums/datum.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package datums - -import "fmt" - -var ( - errNumPointsNegative = fmt.Errorf("numPoints must be positive") -) - -type synTS struct { - id int - data []float64 - - lastIdx int -} - -func (ld *synTS) ID() int { - return ld.id -} - -func (ld *synTS) Size() int { - return len(ld.data) -} - -func (ld *synTS) Data() []float64 { - return ld.data -} - -func (ld *synTS) Get(idx int) float64 { - idx = idx % len(ld.data) - if idx < 0 { - idx += len(ld.data) - } - return ld.data[idx] -} - -func (ld *synTS) Next() float64 { - idx := (ld.lastIdx + 1) % len(ld.data) - if idx < 0 { - idx += len(ld.data) - } - ld.lastIdx = idx - return ld.data[idx] -} - -// NewSyntheticTimeSeris generates a new SyntheticTimeSeris using the provided parameters. -func NewSyntheticTimeSeris(id int, numPoints int, fn TSGenFn) (SyntheticTimeSeries, error) { - if numPoints < 0 { - return nil, errNumPointsNegative - } - data := make([]float64, numPoints) - for i := 0; i < numPoints; i++ { - data[i] = fn(i) - } - return &synTS{ - id: id, - data: data, - lastIdx: -1, - }, nil -} diff --git a/src/m3nsch/datums/registry.go b/src/m3nsch/datums/registry.go deleted file mode 100644 index 1a7a228800..0000000000 --- a/src/m3nsch/datums/registry.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package datums - -type tsRegistry struct { - currentIdx int - numPointsPerDatum int - tsGenMap map[int]TSGenFn -} - -func (reg *tsRegistry) Size() int { - return len(reg.tsGenMap) -} - -func (reg *tsRegistry) Get(i int) SyntheticTimeSeries { - sz := reg.Size() - idx := i % sz - if idx < 0 { - idx = idx + sz - } - datum, err := NewSyntheticTimeSeris(idx, reg.numPointsPerDatum, reg.tsGenMap[idx]) - if err != nil { - panic(err) - } - return datum -} - -// NewDefaultRegistry returns a Registry with default timeseries generators -func NewDefaultRegistry(numPointsPerDatum int) Registry { - reg := &tsRegistry{ - numPointsPerDatum: numPointsPerDatum, - tsGenMap: make(map[int]TSGenFn), - } - reg.init() - return reg -} - -func (reg *tsRegistry) init() { - // identity datum - reg.addGenFn(func(i int) float64 { - return float64(i) - }) - - // square datum - reg.addGenFn(func(i int) float64 { - return float64(i * i) - }) - - // TODO(prateek): make this bigger -} - -func (reg *tsRegistry) addGenFn(f TSGenFn) { - idx := reg.currentIdx - reg.tsGenMap[idx] = f - reg.currentIdx++ -} diff --git a/src/m3nsch/datums/types.go b/src/m3nsch/datums/types.go deleted file mode 100644 index 8ecc32922f..0000000000 --- a/src/m3nsch/datums/types.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package datums - -// TSGenFn represents a pure function used to -// generate synthetic time series' -type TSGenFn func(idx int) float64 - -// SyntheticTimeSeries represents a synthetically generated -// time series -type SyntheticTimeSeries interface { - // ID returns the id of the SyntheticTimeSeries - ID() int - - // Size returns the number of points in the SyntheticTimeSeries - Size() int - - // Data returns data points comprising the SyntheticTimeSeries - Data() []float64 - - // Get(n) returns the nth (circularly wrapped) data point - Get(n int) float64 - - // Next simulates an infinite iterator on the SyntheticTimeSeries - Next() float64 -} - -// Registry is a collection of synthetic time series' -type Registry interface { - // Get(n) returns the nth (wrapped circularly) SyntheticTimeSeries - // known to the Registry. - Get(n int) SyntheticTimeSeries - - // Size returns the number of unique time series' - // the Registry is capable of generating. - Size() int -} diff --git a/src/m3nsch/examples/sample_workload.go b/src/m3nsch/examples/sample_workload.go deleted file mode 100644 index 4bff79b6b5..0000000000 --- a/src/m3nsch/examples/sample_workload.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -import ( - "log" - "time" - - "github.com/m3db/m3/src/m3nsch" - "github.com/m3db/m3/src/m3nsch/coordinator" - "github.com/m3db/m3/src/x/instrument" - - "github.com/pborman/getopt" - "github.com/spf13/viper" - "go.uber.org/zap" - validator "gopkg.in/validator.v2" -) - -type m3nschConfig struct { - Endpoints []string `yaml:"endpoints" validate:"min=1"` - Workload workloadConfig `yaml:"workload"` - TargetZone string `yaml:"targetZone" validate:"nonzero"` - TargetEnv string `yaml:"targetZone" validate:"nonzero"` -} - -type workloadConfig struct { - TimeOffsetMins int `yaml:"timeOffset"` - Namespace string `yaml:"namespace" validate:"nonzero"` - MetricNamePrefix string `yaml:"metricPrefix" validate:"nonzero"` - Cardinality int `yaml:"cardinality" validate:"min=100"` - IngressQPS int `yaml:"ingressQPS" validate:"min=10"` -} - -func (wc workloadConfig) toM3nschType() m3nsch.Workload { - return m3nsch.Workload{ - BaseTime: time.Now().Add(time.Duration(wc.TimeOffsetMins) * time.Minute), - Namespace: wc.Namespace, - MetricPrefix: wc.MetricNamePrefix, - Cardinality: wc.Cardinality, - IngressQPS: wc.IngressQPS, - } -} - -func main() { - var ( - configFile = getopt.StringLong("config", 'c', "", "Configuration file") - token = getopt.StringLong("token", 't', "", "Identifier Token") - duration = getopt.DurationLong("duration", 'd', time.Duration(0), "Workload Duration") - force = getopt.BoolLong("force", 'f', "false", "Force") - ) - getopt.Parse() - if len(*configFile) == 0 || len(*token) == 0 || duration.Seconds() == 0 { - getopt.Usage() - return - } - - logger, err := zap.NewDevelopment() - if err != nil { - log.Fatalf("unable to read configuration file: %v", err) - } - - var ( - iopts = instrument.NewOptions().SetLogger(logger) - opts = coordinator.NewOptions(iopts) - ) - conf, err := readConfiguration(*configFile) - if err != nil { - logger.Fatal("unable to read configuration file", zap.Error(err)) - } - - coord, err := coordinator.New(opts, conf.Endpoints) - if err != nil { - logger.Fatal("unable to create coordinator", zap.Error(err)) - } - - workload := conf.Workload.toM3nschType() - err = coord.Init(*token, workload, *force, conf.TargetZone, conf.TargetEnv) - if err != nil { - logger.Fatal("unable to init coordinator", zap.Error(err)) - } - defer coord.Teardown() - - if err := coord.Start(); err != nil { - logger.Fatal("unable to start coordinator", zap.Error(err)) - } - defer coord.Stop() - - stopTime := time.Now().Add(*duration) - for time.Now().Before(stopTime) { - logger.Sugar().Infof("status [ time = %v ]", time.Now().String()) - statusMap, err := coord.Status() - if err != nil { - logger.Sugar().Fatalf("unable to retrieve status: %v", err) - } - logStatus(logger, statusMap) - time.Sleep(time.Second * 5) - } -} - -func logStatus(logger *zap.Logger, statusMap map[string]m3nsch.AgentStatus) { - for endpoint, status := range statusMap { - token := status.Token - if token == "" { - token = "" - } - logger.Sugar().Infof("[%v] MaxQPS: %d, Status: %v, Token: %v, Workload: %+v", - endpoint, status.MaxQPS, status.Status, token, status.Workload) - } -} - -func readConfiguration(filename string) (m3nschConfig, error) { - viper.SetConfigType("yaml") - viper.SetConfigFile(filename) - - var conf m3nschConfig - if err := viper.ReadInConfig(); err != nil { - return conf, err - } - - if err := viper.Unmarshal(&conf); err != nil { - return conf, err - } - - if err := validator.Validate(conf); err != nil { - return conf, err - } - - return conf, nil -} diff --git a/src/m3nsch/generated/convert/to_api.go b/src/m3nsch/generated/convert/to_api.go deleted file mode 100644 index 487a5efce0..0000000000 --- a/src/m3nsch/generated/convert/to_api.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package convert - -import ( - "fmt" - "time" - - "github.com/m3db/m3/src/m3nsch" - proto "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch" - - gogo_proto "github.com/gogo/protobuf/types" -) - -func toTimeFromProtoTimestamp(t *gogo_proto.Timestamp) time.Time { - return time.Unix(t.Seconds, int64(t.Nanos)) -} - -// ToM3nschWorkload converts a rpc Workload into an equivalent API Workload. -func ToM3nschWorkload(workload *proto.Workload) (m3nsch.Workload, error) { - if workload == nil { - return m3nsch.Workload{}, fmt.Errorf("invalid workload") - } - - return m3nsch.Workload{ - BaseTime: toTimeFromProtoTimestamp(workload.BaseTime), - MetricPrefix: workload.MetricPrefix, - Namespace: workload.Namespace, - Cardinality: int(workload.Cardinality), - IngressQPS: int(workload.IngressQPS), - UniqueAmplifier: workload.UniqueAmplifier, - }, nil -} - -// ToM3nschStatus converts a rpc Status into an equivalent API Status. -func ToM3nschStatus(status proto.Status) (m3nsch.Status, error) { - switch status { - case proto.Status_UNINITIALIZED: - return m3nsch.StatusUninitialized, nil - case proto.Status_INITIALIZED: - return m3nsch.StatusInitialized, nil - case proto.Status_RUNNING: - return m3nsch.StatusRunning, nil - } - return m3nsch.StatusUninitialized, fmt.Errorf("invalid status: %s", status.String()) -} diff --git a/src/m3nsch/generated/convert/to_proto.go b/src/m3nsch/generated/convert/to_proto.go deleted file mode 100644 index d8ad9c8d72..0000000000 --- a/src/m3nsch/generated/convert/to_proto.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package convert - -import ( - "github.com/m3db/m3/src/m3nsch" - proto "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch" - - gogo_proto "github.com/gogo/protobuf/types" -) - -// ToProtoStatus converts an API status into a RPC status. -func ToProtoStatus(status m3nsch.Status) proto.Status { - switch status { - case m3nsch.StatusUninitialized: - return proto.Status_UNINITIALIZED - case m3nsch.StatusInitialized: - return proto.Status_INITIALIZED - case m3nsch.StatusRunning: - return proto.Status_RUNNING - } - return proto.Status_UNKNOWN -} - -// ToProtoWorkload converts an API Workload into a RPC Workload. -func ToProtoWorkload(mw m3nsch.Workload) proto.Workload { - var w proto.Workload - w.BaseTime = &gogo_proto.Timestamp{ - Seconds: mw.BaseTime.Unix(), - Nanos: int32(mw.BaseTime.UnixNano()), - } - w.Cardinality = int32(mw.Cardinality) - w.IngressQPS = int32(mw.IngressQPS) - w.MetricPrefix = mw.MetricPrefix - w.Namespace = mw.Namespace - w.UniqueAmplifier = mw.UniqueAmplifier - return w -} diff --git a/src/m3nsch/generated/mocks/generate.go b/src/m3nsch/generated/mocks/generate.go deleted file mode 100644 index d3eac5cfe3..0000000000 --- a/src/m3nsch/generated/mocks/generate.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package mocks - -//go:generate sh -c "mockgen -package=m3nsch -destination=$GOPATH/src/github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch_pb_mock.go -source=$GOPATH/src/github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch.pb.go" diff --git a/src/m3nsch/generated/proto/generate.go b/src/m3nsch/generated/proto/generate.go deleted file mode 100644 index cc1b436665..0000000000 --- a/src/m3nsch/generated/proto/generate.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -//go:generate sh -c "$GOPATH/src/$PACKAGE/scripts/proto-gen.sh $PACKAGE/src/m3nsch/generated/proto" - -package proto diff --git a/src/m3nsch/generated/proto/m3nsch/m3nsch.pb.go b/src/m3nsch/generated/proto/m3nsch/m3nsch.pb.go deleted file mode 100644 index fb2a214f34..0000000000 --- a/src/m3nsch/generated/proto/m3nsch/m3nsch.pb.go +++ /dev/null @@ -1,2099 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch.proto - -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -/* - Package m3nsch is a generated protocol buffer package. - - It is generated from these files: - github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch.proto - - It has these top-level messages: - StatusRequest - StatusResponse - InitRequest - InitResponse - ModifyRequest - ModifyResponse - StartRequest - StartResponse - StopRequest - StopResponse - Workload -*/ -package m3nsch - -import proto "github.com/gogo/protobuf/proto" -import fmt "fmt" -import math "math" -import google_protobuf "github.com/gogo/protobuf/types" - -import context "golang.org/x/net/context" -import grpc "google.golang.org/grpc" - -import binary "encoding/binary" - -import io "io" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package - -type Status int32 - -const ( - Status_UNKNOWN Status = 0 - Status_UNINITIALIZED Status = 1 - Status_INITIALIZED Status = 2 - Status_RUNNING Status = 3 -) - -var Status_name = map[int32]string{ - 0: "UNKNOWN", - 1: "UNINITIALIZED", - 2: "INITIALIZED", - 3: "RUNNING", -} -var Status_value = map[string]int32{ - "UNKNOWN": 0, - "UNINITIALIZED": 1, - "INITIALIZED": 2, - "RUNNING": 3, -} - -func (x Status) String() string { - return proto.EnumName(Status_name, int32(x)) -} -func (Status) EnumDescriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{0} } - -type StatusRequest struct { -} - -func (m *StatusRequest) Reset() { *m = StatusRequest{} } -func (m *StatusRequest) String() string { return proto.CompactTextString(m) } -func (*StatusRequest) ProtoMessage() {} -func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{0} } - -type StatusResponse struct { - Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=m3nsch.Status" json:"status,omitempty"` - Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"` - MaxQPS int64 `protobuf:"varint,3,opt,name=maxQPS,proto3" json:"maxQPS,omitempty"` - Workload *Workload `protobuf:"bytes,4,opt,name=workload" json:"workload,omitempty"` -} - -func (m *StatusResponse) Reset() { *m = StatusResponse{} } -func (m *StatusResponse) String() string { return proto.CompactTextString(m) } -func (*StatusResponse) ProtoMessage() {} -func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{1} } - -func (m *StatusResponse) GetStatus() Status { - if m != nil { - return m.Status - } - return Status_UNKNOWN -} - -func (m *StatusResponse) GetToken() string { - if m != nil { - return m.Token - } - return "" -} - -func (m *StatusResponse) GetMaxQPS() int64 { - if m != nil { - return m.MaxQPS - } - return 0 -} - -func (m *StatusResponse) GetWorkload() *Workload { - if m != nil { - return m.Workload - } - return nil -} - -type InitRequest struct { - Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` - Workload *Workload `protobuf:"bytes,2,opt,name=workload" json:"workload,omitempty"` - Force bool `protobuf:"varint,3,opt,name=force,proto3" json:"force,omitempty"` - TargetZone string `protobuf:"bytes,4,opt,name=targetZone,proto3" json:"targetZone,omitempty"` - TargetEnv string `protobuf:"bytes,5,opt,name=targetEnv,proto3" json:"targetEnv,omitempty"` -} - -func (m *InitRequest) Reset() { *m = InitRequest{} } -func (m *InitRequest) String() string { return proto.CompactTextString(m) } -func (*InitRequest) ProtoMessage() {} -func (*InitRequest) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{2} } - -func (m *InitRequest) GetToken() string { - if m != nil { - return m.Token - } - return "" -} - -func (m *InitRequest) GetWorkload() *Workload { - if m != nil { - return m.Workload - } - return nil -} - -func (m *InitRequest) GetForce() bool { - if m != nil { - return m.Force - } - return false -} - -func (m *InitRequest) GetTargetZone() string { - if m != nil { - return m.TargetZone - } - return "" -} - -func (m *InitRequest) GetTargetEnv() string { - if m != nil { - return m.TargetEnv - } - return "" -} - -type InitResponse struct { -} - -func (m *InitResponse) Reset() { *m = InitResponse{} } -func (m *InitResponse) String() string { return proto.CompactTextString(m) } -func (*InitResponse) ProtoMessage() {} -func (*InitResponse) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{3} } - -type ModifyRequest struct { - Workload *Workload `protobuf:"bytes,1,opt,name=workload" json:"workload,omitempty"` -} - -func (m *ModifyRequest) Reset() { *m = ModifyRequest{} } -func (m *ModifyRequest) String() string { return proto.CompactTextString(m) } -func (*ModifyRequest) ProtoMessage() {} -func (*ModifyRequest) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{4} } - -func (m *ModifyRequest) GetWorkload() *Workload { - if m != nil { - return m.Workload - } - return nil -} - -type ModifyResponse struct { -} - -func (m *ModifyResponse) Reset() { *m = ModifyResponse{} } -func (m *ModifyResponse) String() string { return proto.CompactTextString(m) } -func (*ModifyResponse) ProtoMessage() {} -func (*ModifyResponse) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{5} } - -type StartRequest struct { -} - -func (m *StartRequest) Reset() { *m = StartRequest{} } -func (m *StartRequest) String() string { return proto.CompactTextString(m) } -func (*StartRequest) ProtoMessage() {} -func (*StartRequest) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{6} } - -type StartResponse struct { -} - -func (m *StartResponse) Reset() { *m = StartResponse{} } -func (m *StartResponse) String() string { return proto.CompactTextString(m) } -func (*StartResponse) ProtoMessage() {} -func (*StartResponse) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{7} } - -type StopRequest struct { -} - -func (m *StopRequest) Reset() { *m = StopRequest{} } -func (m *StopRequest) String() string { return proto.CompactTextString(m) } -func (*StopRequest) ProtoMessage() {} -func (*StopRequest) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{8} } - -type StopResponse struct { -} - -func (m *StopResponse) Reset() { *m = StopResponse{} } -func (m *StopResponse) String() string { return proto.CompactTextString(m) } -func (*StopResponse) ProtoMessage() {} -func (*StopResponse) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{9} } - -type Workload struct { - BaseTime *google_protobuf.Timestamp `protobuf:"bytes,1,opt,name=baseTime" json:"baseTime,omitempty"` - MetricPrefix string `protobuf:"bytes,2,opt,name=metricPrefix,proto3" json:"metricPrefix,omitempty"` - Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` - Cardinality int32 `protobuf:"varint,4,opt,name=cardinality,proto3" json:"cardinality,omitempty"` - IngressQPS int32 `protobuf:"varint,5,opt,name=ingressQPS,proto3" json:"ingressQPS,omitempty"` - UniqueAmplifier float64 `protobuf:"fixed64,6,opt,name=uniqueAmplifier,proto3" json:"uniqueAmplifier,omitempty"` -} - -func (m *Workload) Reset() { *m = Workload{} } -func (m *Workload) String() string { return proto.CompactTextString(m) } -func (*Workload) ProtoMessage() {} -func (*Workload) Descriptor() ([]byte, []int) { return fileDescriptorM3Nsch, []int{10} } - -func (m *Workload) GetBaseTime() *google_protobuf.Timestamp { - if m != nil { - return m.BaseTime - } - return nil -} - -func (m *Workload) GetMetricPrefix() string { - if m != nil { - return m.MetricPrefix - } - return "" -} - -func (m *Workload) GetNamespace() string { - if m != nil { - return m.Namespace - } - return "" -} - -func (m *Workload) GetCardinality() int32 { - if m != nil { - return m.Cardinality - } - return 0 -} - -func (m *Workload) GetIngressQPS() int32 { - if m != nil { - return m.IngressQPS - } - return 0 -} - -func (m *Workload) GetUniqueAmplifier() float64 { - if m != nil { - return m.UniqueAmplifier - } - return 0 -} - -func init() { - proto.RegisterType((*StatusRequest)(nil), "m3nsch.StatusRequest") - proto.RegisterType((*StatusResponse)(nil), "m3nsch.StatusResponse") - proto.RegisterType((*InitRequest)(nil), "m3nsch.InitRequest") - proto.RegisterType((*InitResponse)(nil), "m3nsch.InitResponse") - proto.RegisterType((*ModifyRequest)(nil), "m3nsch.ModifyRequest") - proto.RegisterType((*ModifyResponse)(nil), "m3nsch.ModifyResponse") - proto.RegisterType((*StartRequest)(nil), "m3nsch.StartRequest") - proto.RegisterType((*StartResponse)(nil), "m3nsch.StartResponse") - proto.RegisterType((*StopRequest)(nil), "m3nsch.StopRequest") - proto.RegisterType((*StopResponse)(nil), "m3nsch.StopResponse") - proto.RegisterType((*Workload)(nil), "m3nsch.Workload") - proto.RegisterEnum("m3nsch.Status", Status_name, Status_value) -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// Client API for Mensch service - -type MenschClient interface { - Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) - Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) - Start(ctx context.Context, in *StartRequest, opts ...grpc.CallOption) (*StartResponse, error) - Stop(ctx context.Context, in *StopRequest, opts ...grpc.CallOption) (*StopResponse, error) - Modify(ctx context.Context, in *ModifyRequest, opts ...grpc.CallOption) (*ModifyResponse, error) -} - -type menschClient struct { - cc *grpc.ClientConn -} - -func NewMenschClient(cc *grpc.ClientConn) MenschClient { - return &menschClient{cc} -} - -func (c *menschClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) { - out := new(StatusResponse) - err := grpc.Invoke(ctx, "/m3nsch.Mensch/Status", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *menschClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) { - out := new(InitResponse) - err := grpc.Invoke(ctx, "/m3nsch.Mensch/Init", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *menschClient) Start(ctx context.Context, in *StartRequest, opts ...grpc.CallOption) (*StartResponse, error) { - out := new(StartResponse) - err := grpc.Invoke(ctx, "/m3nsch.Mensch/Start", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *menschClient) Stop(ctx context.Context, in *StopRequest, opts ...grpc.CallOption) (*StopResponse, error) { - out := new(StopResponse) - err := grpc.Invoke(ctx, "/m3nsch.Mensch/Stop", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *menschClient) Modify(ctx context.Context, in *ModifyRequest, opts ...grpc.CallOption) (*ModifyResponse, error) { - out := new(ModifyResponse) - err := grpc.Invoke(ctx, "/m3nsch.Mensch/Modify", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// Server API for Mensch service - -type MenschServer interface { - Status(context.Context, *StatusRequest) (*StatusResponse, error) - Init(context.Context, *InitRequest) (*InitResponse, error) - Start(context.Context, *StartRequest) (*StartResponse, error) - Stop(context.Context, *StopRequest) (*StopResponse, error) - Modify(context.Context, *ModifyRequest) (*ModifyResponse, error) -} - -func RegisterMenschServer(s *grpc.Server, srv MenschServer) { - s.RegisterService(&_Mensch_serviceDesc, srv) -} - -func _Mensch_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MenschServer).Status(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/m3nsch.Mensch/Status", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MenschServer).Status(ctx, req.(*StatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Mensch_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(InitRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MenschServer).Init(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/m3nsch.Mensch/Init", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MenschServer).Init(ctx, req.(*InitRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Mensch_Start_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StartRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MenschServer).Start(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/m3nsch.Mensch/Start", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MenschServer).Start(ctx, req.(*StartRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Mensch_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StopRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MenschServer).Stop(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/m3nsch.Mensch/Stop", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MenschServer).Stop(ctx, req.(*StopRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Mensch_Modify_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ModifyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MenschServer).Modify(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/m3nsch.Mensch/Modify", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MenschServer).Modify(ctx, req.(*ModifyRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _Mensch_serviceDesc = grpc.ServiceDesc{ - ServiceName: "m3nsch.Mensch", - HandlerType: (*MenschServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Status", - Handler: _Mensch_Status_Handler, - }, - { - MethodName: "Init", - Handler: _Mensch_Init_Handler, - }, - { - MethodName: "Start", - Handler: _Mensch_Start_Handler, - }, - { - MethodName: "Stop", - Handler: _Mensch_Stop_Handler, - }, - { - MethodName: "Modify", - Handler: _Mensch_Modify_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch.proto", -} - -func (m *StatusRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StatusRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *StatusResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StatusResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Status != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.Status)) - } - if len(m.Token) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.Token))) - i += copy(dAtA[i:], m.Token) - } - if m.MaxQPS != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.MaxQPS)) - } - if m.Workload != nil { - dAtA[i] = 0x22 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.Workload.Size())) - n1, err := m.Workload.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n1 - } - return i, nil -} - -func (m *InitRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *InitRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Token) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.Token))) - i += copy(dAtA[i:], m.Token) - } - if m.Workload != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.Workload.Size())) - n2, err := m.Workload.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n2 - } - if m.Force { - dAtA[i] = 0x18 - i++ - if m.Force { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if len(m.TargetZone) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.TargetZone))) - i += copy(dAtA[i:], m.TargetZone) - } - if len(m.TargetEnv) > 0 { - dAtA[i] = 0x2a - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.TargetEnv))) - i += copy(dAtA[i:], m.TargetEnv) - } - return i, nil -} - -func (m *InitResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *InitResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *ModifyRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ModifyRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Workload != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.Workload.Size())) - n3, err := m.Workload.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n3 - } - return i, nil -} - -func (m *ModifyResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ModifyResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *StartRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StartRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *StartResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StartResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *StopRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StopRequest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *StopResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StopResponse) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - return i, nil -} - -func (m *Workload) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Workload) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.BaseTime != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.BaseTime.Size())) - n4, err := m.BaseTime.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n4 - } - if len(m.MetricPrefix) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.MetricPrefix))) - i += copy(dAtA[i:], m.MetricPrefix) - } - if len(m.Namespace) > 0 { - dAtA[i] = 0x1a - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(len(m.Namespace))) - i += copy(dAtA[i:], m.Namespace) - } - if m.Cardinality != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.Cardinality)) - } - if m.IngressQPS != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintM3Nsch(dAtA, i, uint64(m.IngressQPS)) - } - if m.UniqueAmplifier != 0 { - dAtA[i] = 0x31 - i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.UniqueAmplifier)))) - i += 8 - } - return i, nil -} - -func encodeVarintM3Nsch(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return offset + 1 -} -func (m *StatusRequest) Size() (n int) { - var l int - _ = l - return n -} - -func (m *StatusResponse) Size() (n int) { - var l int - _ = l - if m.Status != 0 { - n += 1 + sovM3Nsch(uint64(m.Status)) - } - l = len(m.Token) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - if m.MaxQPS != 0 { - n += 1 + sovM3Nsch(uint64(m.MaxQPS)) - } - if m.Workload != nil { - l = m.Workload.Size() - n += 1 + l + sovM3Nsch(uint64(l)) - } - return n -} - -func (m *InitRequest) Size() (n int) { - var l int - _ = l - l = len(m.Token) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - if m.Workload != nil { - l = m.Workload.Size() - n += 1 + l + sovM3Nsch(uint64(l)) - } - if m.Force { - n += 2 - } - l = len(m.TargetZone) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - l = len(m.TargetEnv) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - return n -} - -func (m *InitResponse) Size() (n int) { - var l int - _ = l - return n -} - -func (m *ModifyRequest) Size() (n int) { - var l int - _ = l - if m.Workload != nil { - l = m.Workload.Size() - n += 1 + l + sovM3Nsch(uint64(l)) - } - return n -} - -func (m *ModifyResponse) Size() (n int) { - var l int - _ = l - return n -} - -func (m *StartRequest) Size() (n int) { - var l int - _ = l - return n -} - -func (m *StartResponse) Size() (n int) { - var l int - _ = l - return n -} - -func (m *StopRequest) Size() (n int) { - var l int - _ = l - return n -} - -func (m *StopResponse) Size() (n int) { - var l int - _ = l - return n -} - -func (m *Workload) Size() (n int) { - var l int - _ = l - if m.BaseTime != nil { - l = m.BaseTime.Size() - n += 1 + l + sovM3Nsch(uint64(l)) - } - l = len(m.MetricPrefix) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - l = len(m.Namespace) - if l > 0 { - n += 1 + l + sovM3Nsch(uint64(l)) - } - if m.Cardinality != 0 { - n += 1 + sovM3Nsch(uint64(m.Cardinality)) - } - if m.IngressQPS != 0 { - n += 1 + sovM3Nsch(uint64(m.IngressQPS)) - } - if m.UniqueAmplifier != 0 { - n += 9 - } - return n -} - -func sovM3Nsch(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozM3Nsch(x uint64) (n int) { - return sovM3Nsch(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *StatusRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StatusRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StatusResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StatusResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= (Status(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Token", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Token = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxQPS", wireType) - } - m.MaxQPS = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.MaxQPS |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Workload", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Workload == nil { - m.Workload = &Workload{} - } - if err := m.Workload.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *InitRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: InitRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: InitRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Token", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Token = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Workload", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Workload == nil { - m.Workload = &Workload{} - } - if err := m.Workload.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Force", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Force = bool(v != 0) - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TargetZone", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TargetZone = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TargetEnv", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TargetEnv = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *InitResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: InitResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: InitResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ModifyRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ModifyRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ModifyRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Workload", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Workload == nil { - m.Workload = &Workload{} - } - if err := m.Workload.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ModifyResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ModifyResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ModifyResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StartRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StartRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StartRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StartResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StartResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StartResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StopRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StopRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StopRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StopResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StopResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StopResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Workload) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Workload: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Workload: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BaseTime", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.BaseTime == nil { - m.BaseTime = &google_protobuf.Timestamp{} - } - if err := m.BaseTime.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MetricPrefix", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.MetricPrefix = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthM3Nsch - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Namespace = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Cardinality", wireType) - } - m.Cardinality = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Cardinality |= (int32(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field IngressQPS", wireType) - } - m.IngressQPS = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.IngressQPS |= (int32(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field UniqueAmplifier", wireType) - } - var v uint64 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) - iNdEx += 8 - m.UniqueAmplifier = float64(math.Float64frombits(v)) - default: - iNdEx = preIndex - skippy, err := skipM3Nsch(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthM3Nsch - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipM3Nsch(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - iNdEx += length - if length < 0 { - return 0, ErrInvalidLengthM3Nsch - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowM3Nsch - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipM3Nsch(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthM3Nsch = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowM3Nsch = fmt.Errorf("proto: integer overflow") -) - -func init() { - proto.RegisterFile("github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/m3nsch.proto", fileDescriptorM3Nsch) -} - -var fileDescriptorM3Nsch = []byte{ - // 617 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x93, 0x4d, 0x6e, 0xd3, 0x40, - 0x14, 0xc7, 0x3b, 0x69, 0x63, 0x92, 0x97, 0x26, 0x0d, 0x43, 0x5a, 0x45, 0x11, 0x0a, 0x51, 0x16, - 0x28, 0x42, 0x28, 0x16, 0x0d, 0xa2, 0x2b, 0x16, 0x45, 0x14, 0x14, 0x41, 0x4d, 0x71, 0x5a, 0x55, - 0xea, 0x6e, 0xe2, 0x4c, 0xdc, 0x51, 0x63, 0x8f, 0x3b, 0x33, 0x86, 0xf6, 0x16, 0x2c, 0x10, 0x37, - 0xe0, 0x2e, 0x2c, 0x39, 0x02, 0x2a, 0x17, 0xe0, 0x08, 0xc8, 0x9e, 0x71, 0xec, 0x64, 0xd3, 0x95, - 0xf5, 0x7e, 0xef, 0x73, 0xde, 0xfb, 0x1b, 0x0e, 0x7d, 0xa6, 0x2e, 0xe3, 0xe9, 0xd0, 0xe3, 0x81, - 0x1d, 0x8c, 0x66, 0x53, 0x3b, 0x18, 0xd9, 0x52, 0x78, 0x76, 0x30, 0x0a, 0xa5, 0x77, 0x69, 0xfb, - 0x34, 0xa4, 0x82, 0x28, 0x3a, 0xb3, 0x23, 0xc1, 0x15, 0xcf, 0xb0, 0xfe, 0x0c, 0x53, 0x86, 0x2d, - 0x6d, 0x75, 0x9e, 0xf8, 0x9c, 0xfb, 0x0b, 0xaa, 0x23, 0xa7, 0xf1, 0xdc, 0x56, 0x2c, 0xa0, 0x52, - 0x91, 0x20, 0xd2, 0x81, 0xfd, 0x1d, 0xa8, 0x4f, 0x14, 0x51, 0xb1, 0x74, 0xe9, 0x75, 0x4c, 0xa5, - 0xea, 0x7f, 0x47, 0xd0, 0xc8, 0x88, 0x8c, 0x78, 0x28, 0x29, 0x7e, 0x0a, 0x96, 0x4c, 0x49, 0x1b, - 0xf5, 0xd0, 0xa0, 0xb1, 0xdf, 0x18, 0x9a, 0x5e, 0x26, 0xce, 0x78, 0x71, 0x0b, 0xca, 0x8a, 0x5f, - 0xd1, 0xb0, 0x5d, 0xea, 0xa1, 0x41, 0xd5, 0xd5, 0x06, 0xde, 0x03, 0x2b, 0x20, 0x37, 0x9f, 0x4f, - 0x26, 0xed, 0xcd, 0x1e, 0x1a, 0x6c, 0xba, 0xc6, 0xc2, 0xcf, 0xa1, 0xf2, 0x95, 0x8b, 0xab, 0x05, - 0x27, 0xb3, 0xf6, 0x56, 0x0f, 0x0d, 0x6a, 0xfb, 0xcd, 0xac, 0xee, 0xb9, 0xe1, 0xee, 0x32, 0xa2, - 0xff, 0x13, 0x41, 0x6d, 0x1c, 0x32, 0x65, 0xc6, 0xcc, 0x7b, 0xa1, 0x62, 0xaf, 0x62, 0xcd, 0xd2, - 0x7d, 0x35, 0x93, 0x1a, 0x73, 0x2e, 0x3c, 0x9a, 0x0e, 0x56, 0x71, 0xb5, 0x81, 0xbb, 0x00, 0x8a, - 0x08, 0x9f, 0xaa, 0x0b, 0x1e, 0xd2, 0x74, 0xb2, 0xaa, 0x5b, 0x20, 0xf8, 0x31, 0x54, 0xb5, 0x75, - 0x14, 0x7e, 0x69, 0x97, 0x53, 0x77, 0x0e, 0xfa, 0x0d, 0xd8, 0xd6, 0x63, 0xea, 0xdd, 0xf5, 0x5f, - 0x43, 0xfd, 0x98, 0xcf, 0xd8, 0xfc, 0x36, 0x1b, 0xbc, 0x38, 0x22, 0xba, 0xf7, 0xd9, 0x4d, 0x68, - 0x64, 0xe9, 0xa6, 0x60, 0x03, 0xb6, 0x27, 0x8a, 0x88, 0x6c, 0x11, 0xe6, 0x80, 0x22, 0xef, 0x58, - 0x87, 0xda, 0x44, 0xf1, 0x28, 0xf3, 0xa7, 0xf1, 0x89, 0x69, 0xdc, 0xff, 0x10, 0x54, 0xb2, 0x46, - 0xf8, 0x15, 0x54, 0xa6, 0x44, 0xd2, 0x53, 0x16, 0x50, 0x33, 0x4c, 0x67, 0xa8, 0x15, 0x33, 0xcc, - 0x14, 0x33, 0x3c, 0xcd, 0x14, 0xe3, 0x2e, 0x63, 0x71, 0x1f, 0xb6, 0x03, 0xaa, 0x04, 0xf3, 0x4e, - 0x04, 0x9d, 0xb3, 0x1b, 0x73, 0xf0, 0x15, 0x96, 0xec, 0x29, 0x24, 0x01, 0x95, 0x11, 0x31, 0x1b, - 0xae, 0xba, 0x39, 0xc0, 0x3d, 0xa8, 0x79, 0x44, 0xcc, 0x58, 0x48, 0x16, 0x4c, 0xdd, 0xa6, 0x6b, - 0x2e, 0xbb, 0x45, 0x94, 0xdc, 0x81, 0x85, 0xbe, 0xa0, 0x52, 0x26, 0xda, 0x29, 0xa7, 0x01, 0x05, - 0x82, 0x07, 0xb0, 0x13, 0x87, 0xec, 0x3a, 0xa6, 0x87, 0x41, 0xb4, 0x60, 0x73, 0x46, 0x45, 0xdb, - 0xea, 0xa1, 0x01, 0x72, 0xd7, 0xf1, 0xb3, 0x77, 0x60, 0x69, 0xa5, 0xe2, 0x1a, 0x3c, 0x38, 0x73, - 0x3e, 0x38, 0x9f, 0xce, 0x9d, 0xe6, 0x06, 0x7e, 0x08, 0xf5, 0x33, 0x67, 0xec, 0x8c, 0x4f, 0xc7, - 0x87, 0x1f, 0xc7, 0x17, 0x47, 0x6f, 0x9b, 0x08, 0xef, 0x40, 0xad, 0x08, 0x4a, 0x49, 0x82, 0x7b, - 0xe6, 0x38, 0x63, 0xe7, 0x7d, 0x73, 0x73, 0xff, 0x47, 0x09, 0xac, 0x63, 0x9a, 0x9c, 0x0a, 0x1f, - 0x2c, 0x4b, 0xee, 0xae, 0xfd, 0x0c, 0x7a, 0xed, 0x9d, 0xbd, 0x75, 0x6c, 0xfe, 0xa5, 0x17, 0xb0, - 0x95, 0xe8, 0x03, 0x3f, 0xca, 0xfc, 0x05, 0x51, 0x77, 0x5a, 0xab, 0xd0, 0xa4, 0xbc, 0x84, 0x72, - 0x7a, 0x61, 0xdc, 0x2a, 0xd4, 0x5c, 0x0a, 0xa0, 0xb3, 0xbb, 0x46, 0xf3, 0x46, 0xc9, 0xdd, 0xf3, - 0x46, 0x05, 0x51, 0x74, 0x5a, 0xab, 0xd0, 0xa4, 0x1c, 0x80, 0xa5, 0xc5, 0x96, 0x3f, 0x6a, 0x45, - 0xbb, 0xf9, 0xa3, 0x56, 0x35, 0xf9, 0xa6, 0xf9, 0xeb, 0xae, 0x8b, 0x7e, 0xdf, 0x75, 0xd1, 0x9f, - 0xbb, 0x2e, 0xfa, 0xf6, 0xb7, 0xbb, 0x31, 0xb5, 0x52, 0xf9, 0x8c, 0xfe, 0x07, 0x00, 0x00, 0xff, - 0xff, 0x6a, 0x4c, 0xfe, 0xd7, 0xcb, 0x04, 0x00, 0x00, -} diff --git a/src/m3nsch/generated/proto/m3nsch/m3nsch.proto b/src/m3nsch/generated/proto/m3nsch/m3nsch.proto deleted file mode 100644 index 68ef0b891b..0000000000 --- a/src/m3nsch/generated/proto/m3nsch/m3nsch.proto +++ /dev/null @@ -1,68 +0,0 @@ -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; - -package m3nsch; - -service Mensch { - rpc Status(StatusRequest) returns (StatusResponse); - rpc Init(InitRequest) returns (InitResponse); - rpc Start(StartRequest) returns (StartResponse); - rpc Stop(StopRequest) returns (StopResponse); - rpc Modify(ModifyRequest) returns (ModifyResponse); -} - -enum Status { - UNKNOWN = 0; - UNINITIALIZED = 1; - INITIALIZED = 2; - RUNNING = 3; -} - -message StatusRequest {} - -message StatusResponse { - Status status = 1; - string token = 2; - int64 maxQPS = 3; - Workload workload = 4; -} - -message InitRequest { - string token = 1; - Workload workload = 2; - bool force = 3; - string targetZone = 4; - string targetEnv = 5; -} - -message InitResponse { -} - -message ModifyRequest { - Workload workload = 1; -} - -message ModifyResponse { -} - -message StartRequest { -} - -message StartResponse { -} - -message StopRequest { -} - -message StopResponse{ -} - -message Workload { - google.protobuf.Timestamp baseTime = 1; - string metricPrefix = 2; - string namespace = 3; - int32 cardinality = 4; - int32 ingressQPS = 5; - double uniqueAmplifier = 6; -} diff --git a/src/m3nsch/generated/proto/m3nsch/m3nsch_pb_mock.go b/src/m3nsch/generated/proto/m3nsch/m3nsch_pb_mock.go deleted file mode 100644 index 292a4beffb..0000000000 --- a/src/m3nsch/generated/proto/m3nsch/m3nsch_pb_mock.go +++ /dev/null @@ -1,254 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/m3nsch/generated/proto/m3nsch/pb.go - -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package m3nsch is a generated GoMock package. -package m3nsch - -import ( - "reflect" - - "github.com/golang/mock/gomock" - "golang.org/x/net/context" - "google.golang.org/grpc" -) - -// MockMenschClient is a mock of MenschClient interface -type MockMenschClient struct { - ctrl *gomock.Controller - recorder *MockMenschClientMockRecorder -} - -// MockMenschClientMockRecorder is the mock recorder for MockMenschClient -type MockMenschClientMockRecorder struct { - mock *MockMenschClient -} - -// NewMockMenschClient creates a new mock instance -func NewMockMenschClient(ctrl *gomock.Controller) *MockMenschClient { - mock := &MockMenschClient{ctrl: ctrl} - mock.recorder = &MockMenschClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockMenschClient) EXPECT() *MockMenschClientMockRecorder { - return m.recorder -} - -// Status mocks base method -func (m *MockMenschClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{ctx, in} - for _, a := range opts { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Status", varargs...) - ret0, _ := ret[0].(*StatusResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Status indicates an expected call of Status -func (mr *MockMenschClientMockRecorder) Status(ctx, in interface{}, opts ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{ctx, in}, opts...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockMenschClient)(nil).Status), varargs...) -} - -// Init mocks base method -func (m *MockMenschClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{ctx, in} - for _, a := range opts { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Init", varargs...) - ret0, _ := ret[0].(*InitResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Init indicates an expected call of Init -func (mr *MockMenschClientMockRecorder) Init(ctx, in interface{}, opts ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{ctx, in}, opts...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockMenschClient)(nil).Init), varargs...) -} - -// Start mocks base method -func (m *MockMenschClient) Start(ctx context.Context, in *StartRequest, opts ...grpc.CallOption) (*StartResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{ctx, in} - for _, a := range opts { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Start", varargs...) - ret0, _ := ret[0].(*StartResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Start indicates an expected call of Start -func (mr *MockMenschClientMockRecorder) Start(ctx, in interface{}, opts ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{ctx, in}, opts...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockMenschClient)(nil).Start), varargs...) -} - -// Stop mocks base method -func (m *MockMenschClient) Stop(ctx context.Context, in *StopRequest, opts ...grpc.CallOption) (*StopResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{ctx, in} - for _, a := range opts { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Stop", varargs...) - ret0, _ := ret[0].(*StopResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Stop indicates an expected call of Stop -func (mr *MockMenschClientMockRecorder) Stop(ctx, in interface{}, opts ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{ctx, in}, opts...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockMenschClient)(nil).Stop), varargs...) -} - -// Modify mocks base method -func (m *MockMenschClient) Modify(ctx context.Context, in *ModifyRequest, opts ...grpc.CallOption) (*ModifyResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{ctx, in} - for _, a := range opts { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Modify", varargs...) - ret0, _ := ret[0].(*ModifyResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Modify indicates an expected call of Modify -func (mr *MockMenschClientMockRecorder) Modify(ctx, in interface{}, opts ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{ctx, in}, opts...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Modify", reflect.TypeOf((*MockMenschClient)(nil).Modify), varargs...) -} - -// MockMenschServer is a mock of MenschServer interface -type MockMenschServer struct { - ctrl *gomock.Controller - recorder *MockMenschServerMockRecorder -} - -// MockMenschServerMockRecorder is the mock recorder for MockMenschServer -type MockMenschServerMockRecorder struct { - mock *MockMenschServer -} - -// NewMockMenschServer creates a new mock instance -func NewMockMenschServer(ctrl *gomock.Controller) *MockMenschServer { - mock := &MockMenschServer{ctrl: ctrl} - mock.recorder = &MockMenschServerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockMenschServer) EXPECT() *MockMenschServerMockRecorder { - return m.recorder -} - -// Status mocks base method -func (m *MockMenschServer) Status(arg0 context.Context, arg1 *StatusRequest) (*StatusResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Status", arg0, arg1) - ret0, _ := ret[0].(*StatusResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Status indicates an expected call of Status -func (mr *MockMenschServerMockRecorder) Status(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockMenschServer)(nil).Status), arg0, arg1) -} - -// Init mocks base method -func (m *MockMenschServer) Init(arg0 context.Context, arg1 *InitRequest) (*InitResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0, arg1) - ret0, _ := ret[0].(*InitResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Init indicates an expected call of Init -func (mr *MockMenschServerMockRecorder) Init(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockMenschServer)(nil).Init), arg0, arg1) -} - -// Start mocks base method -func (m *MockMenschServer) Start(arg0 context.Context, arg1 *StartRequest) (*StartResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", arg0, arg1) - ret0, _ := ret[0].(*StartResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Start indicates an expected call of Start -func (mr *MockMenschServerMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockMenschServer)(nil).Start), arg0, arg1) -} - -// Stop mocks base method -func (m *MockMenschServer) Stop(arg0 context.Context, arg1 *StopRequest) (*StopResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stop", arg0, arg1) - ret0, _ := ret[0].(*StopResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Stop indicates an expected call of Stop -func (mr *MockMenschServerMockRecorder) Stop(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockMenschServer)(nil).Stop), arg0, arg1) -} - -// Modify mocks base method -func (m *MockMenschServer) Modify(arg0 context.Context, arg1 *ModifyRequest) (*ModifyResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Modify", arg0, arg1) - ret0, _ := ret[0].(*ModifyResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Modify indicates an expected call of Modify -func (mr *MockMenschServerMockRecorder) Modify(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Modify", reflect.TypeOf((*MockMenschServer)(nil).Modify), arg0, arg1) -} diff --git a/src/m3nsch/types.go b/src/m3nsch/types.go deleted file mode 100644 index b43dc2c315..0000000000 --- a/src/m3nsch/types.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package m3nsch - -import ( - "time" - - "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/x/instrument" - xtime "github.com/m3db/m3/src/x/time" -) - -// Status represents the various states the load generation processes may exist in. -type Status int - -const ( - // StatusUninitialized refers to a load generation process yet to be initialized. - StatusUninitialized Status = iota - - // StatusInitialized refers to a load generation process which has been initialized. - StatusInitialized - - // StatusRunning refers to a load generation process which is running. - StatusRunning -) - -// Workload is a collection of attributes required to define a load generation workload. -// TODO(prateek): add support for duration -type Workload struct { - // BaseTime is the epoch value for time used during load generation, all timestamps are - // generated relative to it. - BaseTime time.Time - - // Namespace is the target namespace to perform the writes against. - Namespace string - - // MetricPrefix is the string prefixed to each metric used. - MetricPrefix string - - // Cardinality is the number of unique metrics used. - Cardinality int - - // IngressQPS is the number of metrics written per second. - IngressQPS int - - // MetricStartIdx is an offset to control metric numbering. Can be safely ignored - // by external callers. - MetricStartIdx int - - // UniqueAmplifier is the percentage of unique metrics generated as a float - // between 0.0 and 1.0 that will be unique. This allows for generating metrics - // with steady cardinality rate over time. - UniqueAmplifier float64 -} - -// Coordinator refers to the process responsible for synchronizing load generation. -type Coordinator interface { - // Status returns the status of the agents known to this process. - Status() (map[string]AgentStatus, error) - - // Init acquires resources required on known agent processes to be able to - // generate load. - // NB(prateek): Init takes ~30s to initialize m3db.Session objects - Init(token string, w Workload, force bool, targetZone string, targetEnv string) error - - // Workload returns the aggregate workload currently initialized on the - // agent processes known to the coordinator process. - Workload() (Workload, error) - - // SetWorkload splits the specified workload into smaller chunks, and distributes them - // across known agent processes. - SetWorkload(Workload) error - - // Start begins the load generation process on known agent processes. - Start() error - - // Stops ends the load generation process on known agent processes. - Stop() error - - // Teardown releases resources held by the Coordinator process (connections, state tracking structures). - Teardown() error -} - -// CoordinatorOptions is a collection of the various knobs to control Coordinator behavior. -type CoordinatorOptions interface { - // SetInstrumentOptions sets the InstrumentOptions - SetInstrumentOptions(instrument.Options) CoordinatorOptions - - // InstrumentOptions returns the set InstrumentOptions - InstrumentOptions() instrument.Options - - // SetTimeout sets the timeout for rpc interaction - SetTimeout(time.Duration) CoordinatorOptions - - // Timeout returns the timeout for rpc interaction - Timeout() time.Duration - - // SetParallelOperations sets a flag determining if the operations - // performed by the coordinator against various endpoints are to be - // parallelized or not - SetParallelOperations(bool) CoordinatorOptions - - // ParallelOperations returns a flag indicating if the operations - // performed by the coordinator against various endpoints are to be - // parallelized or not - ParallelOperations() bool -} - -// AgentStatus is a collection of attributes capturing the state -// of a agent process. -type AgentStatus struct { - // Status refers to the agent process' running status - Status Status - - // Token (if non-empty) is the breadcrumb used to Init the agent process' - Token string - - // MaxQPS is the maximum QPS attainable by the Agent process - MaxQPS int64 - - // Workload is the currently configured workload on the agent process - Workload Workload -} - -// Agent refers to the process responsible for executing load generation. -type Agent interface { - // Status returns the status of the agent process. - Status() AgentStatus - - // Workload returns Workload currently configured on the agent process. - Workload() Workload - - // SetWorkload sets the Workload on the agent process. - SetWorkload(Workload) - - // Init initializes resources required by the agent process. - Init(token string, w Workload, force bool, targetZone string, targetEnv string) error - - // Start begins the load generation process if the agent is Initialized, or errors if it is not. - Start() error - - // Stop ends the load generation process if the agent is Running. - Stop() error - - // TODO(prateek): stats - - // MaxQPS returns the maximum QPS this Agent is capable of driving. - // MaxQPS := `AgentOptions.MaxWorkerQPS() * AgentOptions.Concurrency()` - MaxQPS() int64 -} - -// NewSessionFn creates a new client.Session for the specified environment, zone. -type NewSessionFn func(targetZone string, targetEnv string) (client.Session, error) - -// AgentOptions is a collection of knobs to control Agent behavior. -type AgentOptions interface { - // SetInstrumentOptions sets the InstrumentOptions - SetInstrumentOptions(instrument.Options) AgentOptions - - // InstrumentOptions returns the set InstrumentOptions - InstrumentOptions() instrument.Options - - // SetMaxWorkerQPS sets the maximum QPS per 'worker' go-routine used in - // the load generation process. - SetMaxWorkerQPS(int64) AgentOptions - - // SetMaxWorkerQPS returns the maximum QPS per 'worker' go-routine used in - // the load generation process. - MaxWorkerQPS() int64 - - // SetConcurrency sets the number of concurrent go routines used. - SetConcurrency(int) AgentOptions - - // Concurrency returns the number of concurrent go routines usable during - // load generation. - Concurrency() int - - // SetNewSessionFn sets the new session function - SetNewSessionFn(fn NewSessionFn) AgentOptions - - // NewSessionFn returns the new session function - NewSessionFn() NewSessionFn - - // SetTimeUnit sets the time unit to use during load operations. - SetTimeUnit(xtime.Unit) AgentOptions - - // TimeUnit returns the time unit used during load operations. - TimeUnit() xtime.Unit -} From 27e6a051e90338291fa66cb90cbecb83b34a550a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Fri, 30 Oct 2020 16:56:49 +0200 Subject: [PATCH 12/47] [dbnode] Read only namespaces (#2803) --- src/dbnode/namespace/types.go | 94 ++++++++++++++-------------- src/dbnode/storage/namespace.go | 23 ++++++- src/dbnode/storage/namespace_test.go | 31 +++++++++ src/dbnode/storage/storage_mock.go | 52 +++++++++++++++ src/dbnode/storage/types.go | 6 ++ 5 files changed, 156 insertions(+), 50 deletions(-) diff --git a/src/dbnode/namespace/types.go b/src/dbnode/namespace/types.go index 1096cac58f..39b71a1154 100644 --- a/src/dbnode/namespace/types.go +++ b/src/dbnode/namespace/types.go @@ -32,48 +32,48 @@ import ( "github.com/gogo/protobuf/proto" ) -// Options controls namespace behavior +// Options controls namespace behavior. type Options interface { - // Validate validates the options + // Validate validates the options. Validate() error - // Equal returns true if the provide value is equal to this one + // Equal returns true if the provide value is equal to this one. Equal(value Options) bool - // SetBootstrapEnabled sets whether this namespace requires bootstrapping + // SetBootstrapEnabled sets whether this namespace requires bootstrapping. SetBootstrapEnabled(value bool) Options - // BootstrapEnabled returns whether this namespace requires bootstrapping + // BootstrapEnabled returns whether this namespace requires bootstrapping. BootstrapEnabled() bool - // SetFlushEnabled sets whether the in-memory data for this namespace needs to be flushed + // SetFlushEnabled sets whether the in-memory data for this namespace needs to be flushed. SetFlushEnabled(value bool) Options - // FlushEnabled returns whether the in-memory data for this namespace needs to be flushed + // FlushEnabled returns whether the in-memory data for this namespace needs to be flushed. FlushEnabled() bool - // SetSnapshotEnabled sets whether the in-memory data for this namespace should be snapshotted regularly + // SetSnapshotEnabled sets whether the in-memory data for this namespace should be snapshotted regularly. SetSnapshotEnabled(value bool) Options - // SnapshotEnabled returns whether the in-memory data for this namespace should be snapshotted regularly + // SnapshotEnabled returns whether the in-memory data for this namespace should be snapshotted regularly. SnapshotEnabled() bool - // SetWritesToCommitLog sets whether writes for series in this namespace need to go to commit log + // SetWritesToCommitLog sets whether writes for series in this namespace need to go to commit log. SetWritesToCommitLog(value bool) Options - // WritesToCommitLog returns whether writes for series in this namespace need to go to commit log + // WritesToCommitLog returns whether writes for series in this namespace need to go to commit log. WritesToCommitLog() bool - // SetCleanupEnabled sets whether this namespace requires cleaning up fileset/snapshot files + // SetCleanupEnabled sets whether this namespace requires cleaning up fileset/snapshot files. SetCleanupEnabled(value bool) Options - // CleanupEnabled returns whether this namespace requires cleaning up fileset/snapshot files + // CleanupEnabled returns whether this namespace requires cleaning up fileset/snapshot files. CleanupEnabled() bool - // SetRepairEnabled sets whether the data for this namespace needs to be repaired + // SetRepairEnabled sets whether the data for this namespace needs to be repaired. SetRepairEnabled(value bool) Options - // RepairEnabled returns whether the data for this namespace needs to be repaired + // RepairEnabled returns whether the data for this namespace needs to be repaired. RepairEnabled() bool // SetColdWritesEnabled sets whether cold writes are enabled for this namespace. @@ -90,10 +90,10 @@ type Options interface { // CacheBlocksOnRetrieve returns whether to cache blocks from this namespace when retrieved. CacheBlocksOnRetrieve() bool - // SetRetentionOptions sets the retention options for this namespace + // SetRetentionOptions sets the retention options for this namespace. SetRetentionOptions(value retention.Options) Options - // RetentionOptions returns the retention options for this namespace + // RetentionOptions returns the retention options for this namespace. RetentionOptions() retention.Options // SetIndexOptions sets the IndexOptions. @@ -192,15 +192,15 @@ type SchemaListener interface { // and where schema is retrieved from at series read and write path. type SchemaRegistry interface { // GetLatestSchema gets the latest schema for the namespace. - // If proto is not enabled, nil, nil is returned + // If proto is not enabled, nil, nil is returned. GetLatestSchema(id ident.ID) (SchemaDescr, error) // GetSchema gets the latest schema for the namespace. - // If proto is not enabled, nil, nil is returned + // If proto is not enabled, nil, nil is returned. GetSchema(id ident.ID, schemaID string) (SchemaDescr, error) // SetSchemaHistory sets the schema history for the namespace. - // If proto is not enabled, nil is returned + // If proto is not enabled, nil is returned. SetSchemaHistory(id ident.ID, history SchemaHistory) error // RegisterListener registers a schema listener for the namespace. @@ -211,83 +211,81 @@ type SchemaRegistry interface { Close() } -// Metadata represents namespace metadata information +// Metadata represents namespace metadata information. type Metadata interface { - // Equal returns true if the provide value is equal to this one + // Equal returns true if the provided value is equal to this one. Equal(value Metadata) bool - // ID is the ID of the namespace + // ID is the ID of the namespace. ID() ident.ID - // Options is the namespace options + // Options is the namespace options. Options() Options } -// Map is mapping from known namespaces' ID to their Metadata +// Map is mapping from known namespaces' ID to their Metadata. type Map interface { - // Equal returns true if the provide value is equal to this one + // Equal returns true if the provide value is equal to this one. Equal(value Map) bool - // Get gets the metadata for the provided namespace + // Get gets the metadata for the provided namespace. Get(ident.ID) (Metadata, error) - // IDs returns the ID of known namespaces + // IDs returns the ID of known namespaces. IDs() []ident.ID - // Metadatas returns the metadata of known namespaces + // Metadatas returns the metadata of known namespaces. Metadatas() []Metadata } -// Watch is a watch on a namespace Map +// Watch is a watch on a namespace Map. type Watch interface { - // C is the notification channel for when a value becomes available + // C is the notification channel for when a value becomes available. C() <-chan struct{} - // Get the current namespace map + // Get the current namespace map. Get() Map - // Close closes the watch + // Close closes the watch. Close() error } -// Registry is an un-changing container for a Map +// Registry is an un-changing container for a Map. type Registry interface { - // Watch for the Registry changes + // Watch for the Registry changes. Watch() (Watch, error) - // Close closes the registry + // Close closes the registry. Close() error } -// Initializer can init new instances of namespace registries +// Initializer can init new instances of namespace registries. type Initializer interface { - // Init will return a new Registry + // Init will return a new Registry. Init() (Registry, error) } -// DynamicOptions is a set of options for dynamic namespace registry +// DynamicOptions is a set of options for dynamic namespace registry. type DynamicOptions interface { - // Validate validates the options + // Validate validates the options. Validate() error - // SetInstrumentOptions sets the instrumentation options + // SetInstrumentOptions sets the instrumentation options. SetInstrumentOptions(value instrument.Options) DynamicOptions - // InstrumentOptions returns the instrumentation options + // InstrumentOptions returns the instrumentation options. InstrumentOptions() instrument.Options - // SetConfigServiceClient sets the client of ConfigService + // SetConfigServiceClient sets the client of ConfigService. SetConfigServiceClient(c client.Client) DynamicOptions - // ConfigServiceClient returns the client of ConfigService + // ConfigServiceClient returns the client of ConfigService. ConfigServiceClient() client.Client - // SetNamespaceRegistryKey sets the kv-store key used for the - // NamespaceRegistry + // SetNamespaceRegistryKey sets the kv-store key used for the NamespaceRegistry. SetNamespaceRegistryKey(k string) DynamicOptions - // NamespaceRegistryKey returns the kv-store key used for the - // NamespaceRegistry + // NamespaceRegistryKey returns the kv-store key used for the NamespaceRegistry. NamespaceRegistryKey() string // SetForceColdWritesEnabled sets whether or not to force enable cold writes diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index 3b304e2bed..31b8148a1d 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -60,6 +60,7 @@ import ( var ( errNamespaceAlreadyClosed = errors.New("namespace already closed") errNamespaceIndexingDisabled = errors.New("namespace indexing is disabled") + errNamespaceReadOnly = errors.New("cannot write to a read only namespace") ) type commitLogWriter interface { @@ -104,6 +105,7 @@ type dbNamespace struct { sync.RWMutex closed bool + readOnly bool shutdownCh chan struct{} id ident.ID shardSet sharding.ShardSet @@ -677,6 +679,10 @@ func (n *dbNamespace) Write( unit xtime.Unit, annotation []byte, ) (SeriesWrite, error) { + if n.ReadOnly() { + return SeriesWrite{}, errNamespaceReadOnly + } + callStart := n.nowFn() shard, nsCtx, err := n.shardFor(id) if err != nil { @@ -702,6 +708,10 @@ func (n *dbNamespace) WriteTagged( unit xtime.Unit, annotation []byte, ) (SeriesWrite, error) { + if n.ReadOnly() { + return SeriesWrite{}, errNamespaceReadOnly + } + callStart := n.nowFn() if n.reverseIndex == nil { n.metrics.writeTagged.ReportError(n.nowFn().Sub(callStart)) @@ -1140,7 +1150,7 @@ func (n *dbNamespace) WarmFlush( nsCtx := n.nsContextWithRLock() n.RUnlock() - if !n.nopts.FlushEnabled() { + if n.ReadOnly() || !n.nopts.FlushEnabled() { n.metrics.flushWarmData.ReportSuccess(n.nowFn().Sub(callStart)) return nil } @@ -1256,7 +1266,8 @@ func (n *dbNamespace) ColdFlush(flushPersist persist.FlushPreparer) error { // If repair is enabled we still need cold flush regardless of whether cold writes is // enabled since repairs are dependent on the cold flushing logic. - if !n.nopts.ColdWritesEnabled() && !n.nopts.RepairEnabled() { + enabled := n.nopts.ColdWritesEnabled() || n.nopts.RepairEnabled() + if n.ReadOnly() || !enabled { n.metrics.flushColdData.ReportSuccess(n.nowFn().Sub(callStart)) return nil } @@ -1584,6 +1595,14 @@ func (n *dbNamespace) Index() (NamespaceIndex, error) { return n.reverseIndex, nil } +func (n *dbNamespace) ReadOnly() bool { + return n.readOnly +} + +func (n *dbNamespace) SetReadOnly(value bool) { + n.readOnly = value +} + func (n *dbNamespace) shardFor(id ident.ID) (databaseShard, namespace.Context, error) { n.RLock() nsCtx := n.nsContextWithRLock() diff --git a/src/dbnode/storage/namespace_test.go b/src/dbnode/storage/namespace_test.go index c6e135868b..dbff29da6c 100644 --- a/src/dbnode/storage/namespace_test.go +++ b/src/dbnode/storage/namespace_test.go @@ -202,6 +202,27 @@ func TestNamespaceWriteShardNotOwned(t *testing.T) { require.False(t, seriesWrite.WasWritten) } +func TestNamespaceReadOnlyRejectWrites(t *testing.T) { + ctx := context.NewContext() + defer ctx.Close() + + ns, closer := newTestNamespace(t) + defer closer() + + ns.SetReadOnly(true) + + id := ident.StringID("foo") + now := time.Now() + + seriesWrite, err := ns.Write(ctx, id, now, 0, xtime.Second, nil) + require.EqualError(t, err, errNamespaceReadOnly.Error()) + require.False(t, seriesWrite.WasWritten) + + seriesWrite, err = ns.WriteTagged(ctx, id, ident.EmptyTagIterator, now, 0, xtime.Second, nil) + require.EqualError(t, err, errNamespaceReadOnly.Error()) + require.False(t, seriesWrite.WasWritten) +} + func TestNamespaceWriteShardOwned(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -449,6 +470,16 @@ func TestNamespaceFlushDontNeedFlush(t *testing.T) { require.NoError(t, ns.ColdFlush(nil)) } +func TestNamespaceSkipFlushIfReadOnly(t *testing.T) { + ns, closer := newTestNamespace(t) + defer closer() + + ns.bootstrapState = Bootstrapped + ns.SetReadOnly(true) + require.NoError(t, ns.WarmFlush(time.Now(), nil)) + require.NoError(t, ns.ColdFlush(nil)) +} + func TestNamespaceFlushSkipFlushed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/src/dbnode/storage/storage_mock.go b/src/dbnode/storage/storage_mock.go index a312eef2f6..feb8fbef7f 100644 --- a/src/dbnode/storage/storage_mock.go +++ b/src/dbnode/storage/storage_mock.go @@ -1154,6 +1154,32 @@ func (mr *MockNamespaceMockRecorder) StorageOptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageOptions", reflect.TypeOf((*MockNamespace)(nil).StorageOptions)) } +// ReadOnly mocks base method +func (m *MockNamespace) ReadOnly() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadOnly") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ReadOnly indicates an expected call of ReadOnly +func (mr *MockNamespaceMockRecorder) ReadOnly() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadOnly", reflect.TypeOf((*MockNamespace)(nil).ReadOnly)) +} + +// SetReadOnly mocks base method +func (m *MockNamespace) SetReadOnly(value bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetReadOnly", value) +} + +// SetReadOnly indicates an expected call of SetReadOnly +func (mr *MockNamespaceMockRecorder) SetReadOnly(value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReadOnly", reflect.TypeOf((*MockNamespace)(nil).SetReadOnly), value) +} + // MockdatabaseNamespace is a mock of databaseNamespace interface type MockdatabaseNamespace struct { ctrl *gomock.Controller @@ -1304,6 +1330,32 @@ func (mr *MockdatabaseNamespaceMockRecorder) StorageOptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageOptions", reflect.TypeOf((*MockdatabaseNamespace)(nil).StorageOptions)) } +// ReadOnly mocks base method +func (m *MockdatabaseNamespace) ReadOnly() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadOnly") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ReadOnly indicates an expected call of ReadOnly +func (mr *MockdatabaseNamespaceMockRecorder) ReadOnly() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadOnly", reflect.TypeOf((*MockdatabaseNamespace)(nil).ReadOnly)) +} + +// SetReadOnly mocks base method +func (m *MockdatabaseNamespace) SetReadOnly(value bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetReadOnly", value) +} + +// SetReadOnly indicates an expected call of SetReadOnly +func (mr *MockdatabaseNamespaceMockRecorder) SetReadOnly(value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReadOnly", reflect.TypeOf((*MockdatabaseNamespace)(nil).SetReadOnly), value) +} + // Close mocks base method func (m *MockdatabaseNamespace) Close() error { m.ctrl.T.Helper() diff --git a/src/dbnode/storage/types.go b/src/dbnode/storage/types.go index f393ef719e..d9aa6cb52a 100644 --- a/src/dbnode/storage/types.go +++ b/src/dbnode/storage/types.go @@ -291,6 +291,12 @@ type Namespace interface { // StorageOptions returns storage options. StorageOptions() Options + + // ReadOnly returns true if this Namespace is read only. + ReadOnly() bool + + // SetReadOnly sets the value of ReadOnly option. + SetReadOnly(value bool) } // NamespacesByID is a sortable slice of namespaces by ID. From 81836cd33b55098646aa91c97061eb388f71a5be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=BBinas?= Date: Fri, 30 Oct 2020 17:22:18 +0200 Subject: [PATCH 13/47] [query] count_values label naming fixes (#2805) * fixed three failing count_values tests * fix unit tests. cleaned up some code. * revert old code * resolves string parameters correctly by unwrapping parents * Moved IsValid() func to tags.go. unwrapParenExpr() used in parser for pql.Call as well (same as in prometheus implementation). Small code cleanup after review. --- .../functions/aggregation/count_values.go | 7 +++- .../aggregation/count_values_test.go | 23 ++++++++++--- src/query/models/tags.go | 15 ++++++++ src/query/parser/promql/matchers.go | 29 +++++++++++++++- src/query/parser/promql/parse.go | 4 +++ .../parser/promql/resolve_scalar_test.go | 30 ++++++++++++++++ .../compatibility/testdata/aggregators.test | 34 +++++++++---------- 7 files changed, 117 insertions(+), 25 deletions(-) diff --git a/src/query/functions/aggregation/count_values.go b/src/query/functions/aggregation/count_values.go index ee3320e838..97e2c25547 100644 --- a/src/query/functions/aggregation/count_values.go +++ b/src/query/functions/aggregation/count_values.go @@ -161,6 +161,11 @@ func (n *countValuesNode) ProcessBlock( } params := n.op.params + labelName := params.StringParameter + if !models.IsValid(labelName) { + return nil, fmt.Errorf("invalid label name %q", labelName) + } + seriesMetas := utils.FlattenMetadata(meta, stepIter.SeriesMeta()) buckets, metas := utils.GroupSeries( params.MatchingTags, @@ -207,7 +212,7 @@ func (n *countValuesNode) ProcessBlock( blockMetas[v+previousBucketBlockIndex] = block.SeriesMeta{ Name: []byte(n.op.opType), Tags: metas[bucketIndex].Tags.Clone().AddTag(models.Tag{ - Name: []byte(n.op.params.StringParameter), + Name: []byte(labelName), Value: utils.FormatFloatToBytes(k), }), } diff --git a/src/query/functions/aggregation/count_values_test.go b/src/query/functions/aggregation/count_values_test.go index 7404cedb7b..f6e87d30a3 100644 --- a/src/query/functions/aggregation/count_values_test.go +++ b/src/query/functions/aggregation/count_values_test.go @@ -105,7 +105,7 @@ var ( ) func TestSimpleProcessCountValuesFunctionUnfiltered(t *testing.T) { - tagName := "tag-name" + tagName := "tag_name" op, err := NewCountValuesOp(CountValuesType, NodeParams{ StringParameter: tagName, }) @@ -123,7 +123,7 @@ func TestSimpleProcessCountValuesFunctionUnfiltered(t *testing.T) { } func TestSimpleProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { - tagName := "tag-name" + tagName := "tag_name" op, err := NewCountValuesOp(CountValuesType, NodeParams{ MatchingTags: [][]byte{[]byte("a")}, Without: true, StringParameter: tagName, }) @@ -147,7 +147,7 @@ func TestSimpleProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { } func TestCustomProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { - tagName := "tag-name" + tagName := "tag_name" op, err := NewCountValuesOp(CountValuesType, NodeParams{ MatchingTags: [][]byte{[]byte("a")}, Without: true, StringParameter: tagName, }) @@ -199,7 +199,7 @@ func TestCustomProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { } func TestSimpleProcessCountValuesFunctionFilteringWithA(t *testing.T) { - tagName := "0_tag-name" + tagName := "tag_name_0" op, err := NewCountValuesOp(CountValuesType, NodeParams{ MatchingTags: [][]byte{[]byte("a")}, Without: false, StringParameter: tagName, }) @@ -217,7 +217,7 @@ func TestSimpleProcessCountValuesFunctionFilteringWithA(t *testing.T) { } func TestProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { - tagName := "tag-name" + tagName := "tag_name" op, err := NewCountValuesOp(CountValuesType, NodeParams{ MatchingTags: [][]byte{[]byte("a")}, Without: true, StringParameter: tagName, }) @@ -301,3 +301,16 @@ func TestProcessCountValuesFunctionFilteringWithoutA(t *testing.T) { assert.Equal(t, ex.Tags, sink.Meta.Tags.Tags) test.CompareValues(t, sink.Metas, tagsToSeriesMeta(expectedTags), sink.Values, expected) } + +func TestShouldFailWhenInvalidLabelName(t *testing.T) { + tagName := "tag-name" + op, _ := NewCountValuesOp(CountValuesType, NodeParams{ + StringParameter: tagName, + }) + bl := test.NewBlockFromValuesWithSeriesMeta(bounds, simpleMetas, simpleVals) + c, _ := executor.NewControllerWithSink(parser.NodeID(1)) + node := op.(countValuesOp).Node(c, transform.Options{}) + err := node.Process(models.NoopQueryContext(), parser.NodeID(0), bl) + require.Error(t, err) +} + diff --git a/src/query/models/tags.go b/src/query/models/tags.go index f768e89215..5f20e84884 100644 --- a/src/query/models/tags.go +++ b/src/query/models/tags.go @@ -55,6 +55,21 @@ func EmptyTags() Tags { return NewTags(0, nil) } +// IsValid is true if the label name matches the pattern of "^[a-zA-Z_][a-zA-Z0-9_]*$". This +// method, however, does not use regex for the check but a much faster +// hardcoded implementation. +func IsValid(ln string) bool { + if len(ln) == 0 { + return false + } + for i, b := range ln { + if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { + return false + } + } + return true +} + // LastComputedID returns the last computed ID; this should only be // used when it is guaranteed that no tag transforms take place between calls. func (t *Tags) LastComputedID() []byte { diff --git a/src/query/parser/promql/matchers.go b/src/query/parser/promql/matchers.go index a0d99e9262..50b965761c 100644 --- a/src/query/parser/promql/matchers.go +++ b/src/query/parser/promql/matchers.go @@ -105,7 +105,12 @@ func NewAggregationOperator(expr *promql.AggregateExpr) (parser.Params, error) { return aggregation.NewTakeOp(op, nodeInformation) case aggregation.CountValuesType: - nodeInformation.StringParameter = expr.Param.String() + paren := unwrapParenExpr(expr.Param) + val, err := resolveStringArgument(paren) + if err != nil { + return nil, err + } + nodeInformation.StringParameter = val return aggregation.NewCountValuesOp(op, nodeInformation) case aggregation.QuantileType: @@ -119,6 +124,28 @@ func NewAggregationOperator(expr *promql.AggregateExpr) (parser.Params, error) { return aggregation.NewAggregationOp(op, nodeInformation) } +func unwrapParenExpr(expr promql.Expr) promql.Expr { + for { + if paren, ok := expr.(*promql.ParenExpr); ok { + expr = paren.Expr + } else { + return expr + } + } +} + +func resolveStringArgument(expr promql.Expr) (string, error) { + if expr == nil { + return "", fmt.Errorf("expression is nil") + } + + if str, ok := expr.(*promql.StringLiteral); ok { + return str.Val, nil + } + + return expr.String(), nil +} + func getAggOpType(opType promql.ItemType) string { switch opType { case promql.SUM: diff --git a/src/query/parser/promql/parse.go b/src/query/parser/promql/parse.go index 9789e2307e..78ccf99af3 100644 --- a/src/query/parser/promql/parse.go +++ b/src/query/parser/promql/parse.go @@ -252,6 +252,10 @@ func (p *parseState) walk(node pql.Node) error { return nil } + for i, expr := range n.Args { + n.Args[i] = unwrapParenExpr(expr) + } + var ( // argTypes describes Prom's expected argument types for this call. argTypes = n.Func.ArgTypes diff --git a/src/query/parser/promql/resolve_scalar_test.go b/src/query/parser/promql/resolve_scalar_test.go index 46abd5764b..d6ead0ea43 100644 --- a/src/query/parser/promql/resolve_scalar_test.go +++ b/src/query/parser/promql/resolve_scalar_test.go @@ -28,6 +28,7 @@ import ( "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,3 +86,32 @@ func TestScalarResolver(t *testing.T) { }) } } + +var stringResolverTests = []struct { + funcString string + expected string +}{ + {`((("value")))`, "value"}, + {`"value"`, "value"}, +} + +func TestStringResolver(t *testing.T) { + for _, tt := range stringResolverTests { + t.Run(tt.funcString, func(t *testing.T) { + parsed, err := Parse(tt.funcString, time.Second, + models.NewTagOptions(), NewParseOptions()) + require.NoError(t, err) + expr := parsed.(*promParser).expr + e := unwrapParenExpr(expr) + label, err := resolveStringArgument(e) + require.NoError(t, err) + assert.Equal(t, tt.expected, label) + }) + } +} + +func TestResolveStringWhenExprIsNil(t *testing.T) { + _, err := resolveStringArgument(nil) + require.Error(t, err) +} + diff --git a/src/query/test/compatibility/testdata/aggregators.test b/src/query/test/compatibility/testdata/aggregators.test index 218a062da1..8eb9cbcc7e 100644 --- a/src/query/test/compatibility/testdata/aggregators.test +++ b/src/query/test/compatibility/testdata/aggregators.test @@ -238,24 +238,22 @@ load 5m version{job="app-server", instance="0", group="canary"} 7 version{job="app-server", instance="1", group="canary"} 7 -# FAILING issue #14 (count_values) -#eval instant at 5m count_values("version", version) -# {version="6"} 5 -# {version="7"} 2 -# {version="8"} 2 - - -#eval instant at 5m count_values(((("version"))), version) -# {version="6"} 5 -# {version="7"} 2 -# {version="8"} 2 - - -#eval instant at 5m count_values without (instance)("version", version) -# {job="api-server", group="production", version="6"} 3 -# {job="api-server", group="canary", version="8"} 2 -# {job="app-server", group="production", version="6"} 2 -# {job="app-server", group="canary", version="7"} 2 +eval instant at 5m count_values("version", version) + {version="6"} 5 + {version="7"} 2 + {version="8"} 2 + +eval instant at 5m count_values(((("version"))), version) + {version="6"} 5 + {version="7"} 2 + {version="8"} 2 + + +eval instant at 5m count_values without (instance)("version", version) + {job="api-server", group="production", version="6"} 3 + {job="api-server", group="canary", version="8"} 2 + {job="app-server", group="production", version="6"} 2 + {job="app-server", group="canary", version="7"} 2 # Overwrite label with output. Don't do this. #eval instant at 5m count_values without (instance)("job", version) From 2bb865a2ff55b7b27a52862e1ef3f260252fba41 Mon Sep 17 00:00:00 2001 From: Alex Bublichenko <46664526+abliqo@users.noreply.github.com> Date: Fri, 30 Oct 2020 14:31:59 -0400 Subject: [PATCH 14/47] [aggregator] Handle follower canLead() for not yet flushed windows (#2818) --- .../aggregator/follower_flush_mgr.go | 133 ++++++++++++++---- .../aggregator/follower_flush_mgr_test.go | 109 ++++++++++++++ 2 files changed, 216 insertions(+), 26 deletions(-) diff --git a/src/aggregator/aggregator/follower_flush_mgr.go b/src/aggregator/aggregator/follower_flush_mgr.go index 860e268e78..ac69dfb4c3 100644 --- a/src/aggregator/aggregator/follower_flush_mgr.go +++ b/src/aggregator/aggregator/follower_flush_mgr.go @@ -38,6 +38,7 @@ type standardFollowerFlusherMetrics struct { resolutionNotFound tally.Counter kvUpdates tally.Counter flushWindowsNotEnded tally.Counter + windowsNeverFlushed tally.Counter } func newStandardFlusherMetrics(scope tally.Scope) standardFollowerFlusherMetrics { @@ -46,6 +47,7 @@ func newStandardFlusherMetrics(scope tally.Scope) standardFollowerFlusherMetrics resolutionNotFound: scope.Counter("resolution-not-found"), kvUpdates: scope.Counter("kv-updates"), flushWindowsNotEnded: scope.Counter("flush-windows-not-ended"), + windowsNeverFlushed: scope.Counter("windows-never-flushed"), } } @@ -56,6 +58,7 @@ type forwardedFollowerFlusherMetrics struct { numForwardedTimesNotFound tally.Counter kvUpdates tally.Counter flushWindowsNotEnded tally.Counter + windowNeverFlushed tally.Counter } func newForwardedFlusherMetrics(scope tally.Scope) forwardedFollowerFlusherMetrics { @@ -66,6 +69,7 @@ func newForwardedFlusherMetrics(scope tally.Scope) forwardedFollowerFlusherMetri numForwardedTimesNotFound: scope.Counter("num-forwarded-times-not-found"), kvUpdates: scope.Counter("kv-updates"), flushWindowsNotEnded: scope.Counter("flush-windows-not-ended"), + windowNeverFlushed: scope.Counter("windows-never-flushed"), } } @@ -226,6 +230,7 @@ func (mgr *followerFlushManager) CanLead() bool { mgr.RLock() defer mgr.RUnlock() + now := mgr.nowFn() if !mgr.electionManager.IsCampaigning() { mgr.metrics.notCampaigning.Inc(1) return false @@ -234,7 +239,7 @@ func (mgr *followerFlushManager) CanLead() bool { return false } - for _, shardFlushTimes := range mgr.processed.ByShard { + for shardID, shardFlushTimes := range mgr.processed.ByShard { // If the shard is tombstoned, there is no need to examine its flush times. if shardFlushTimes.Tombstoned { continue @@ -243,36 +248,32 @@ func (mgr *followerFlushManager) CanLead() bool { // Check that for standard metrics, all the open windows containing the process // start time are closed, meaning the standard metrics that didn't make to the // process have been flushed successfully downstream. - if !mgr.canLead(shardFlushTimes.StandardByResolution, mgr.metrics.standard) { + if !mgr.canLead( + now, + standardMetricListType, + int(shardID), + shardFlushTimes.StandardByResolution, + mgr.metrics.standard, + ) { return false } - if !mgr.canLead(shardFlushTimes.TimedByResolution, mgr.metrics.timed) { + if !mgr.canLead( + now, + timedMetricListType, + int(shardID), + shardFlushTimes.TimedByResolution, + mgr.metrics.timed, + ) { return false } - // Check that the forwarded metrics have been flushed past the process start - // time, meaning the forwarded metrics that didn't make to the process have been - // flushed successfully downstream. - for windowNanos, fbr := range shardFlushTimes.ForwardedByResolution { - if fbr == nil { - mgr.metrics.forwarded.nilForwardedTimes.Inc(1) - return false - } - // Since the timestamps of the forwarded metrics are aligned to the resolution - // boundaries, we simply need to make sure that all forwarded metrics in or before - // the window containing the process start time have been flushed to assert that - // the process can safely take over leadership. - windowSize := time.Duration(windowNanos) - waitTillFlushedTime := mgr.openedAt.Truncate(windowSize) - if waitTillFlushedTime.Equal(mgr.openedAt) { - waitTillFlushedTime = waitTillFlushedTime.Add(-windowSize) - } - for _, lastFlushedNanos := range fbr.ByNumForwardedTimes { - if lastFlushedNanos <= waitTillFlushedTime.UnixNano() { - mgr.metrics.forwarded.flushWindowsNotEnded.Inc(1) - return false - } - } + if !mgr.canLeadForwarded( + now, + int(shardID), + shardFlushTimes.ForwardedByResolution, + mgr.metrics.forwarded, + ) { + return false } } @@ -280,11 +281,29 @@ func (mgr *followerFlushManager) CanLead() bool { } func (mgr *followerFlushManager) canLead( + now time.Time, + flusherType metricListType, + shardID int, flushTimes map[int64]int64, metrics standardFollowerFlusherMetrics, ) bool { for windowNanos, lastFlushedNanos := range flushTimes { windowSize := time.Duration(windowNanos) + if lastFlushedNanos == 0 { + mgr.logger.Info("Encountered a window that was never flushed by leader", + zap.Time("now", now), + zap.Stringer("windowSize", windowSize), + zap.Stringer("flusherType", flusherType), + zap.Int("shardID", int(shardID))) + mgr.metrics.forwarded.windowNeverFlushed.Inc(1) + + if mgr.canLeadNotFlushed(now, windowSize) { + continue + } else { + return false + } + } + windowEndAt := mgr.openedAt.Truncate(windowSize) if windowEndAt.Before(mgr.openedAt) { windowEndAt = windowEndAt.Add(windowSize) @@ -297,6 +316,68 @@ func (mgr *followerFlushManager) canLead( return true } +func (mgr *followerFlushManager) canLeadForwarded( + now time.Time, + shardID int, + flushTimes map[int64]*schema.ForwardedFlushTimesForResolution, + metrics forwardedFollowerFlusherMetrics, +) bool { + // Check that the forwarded metrics have been flushed past the process start + // time, meaning the forwarded metrics that didn't make to the process have been + // flushed successfully downstream. + for windowNanos, fbr := range flushTimes { + if fbr == nil { + mgr.logger.Warn("ForwardedByResolution is nil", + zap.Int("shardID", int(shardID))) + mgr.metrics.forwarded.nilForwardedTimes.Inc(1) + return false + } + // Since the timestamps of the forwarded metrics are aligned to the resolution + // boundaries, we simply need to make sure that all forwarded metrics in or before + // the window containing the process start time have been flushed to assert that + // the process can safely take over leadership. + windowSize := time.Duration(windowNanos) + waitTillFlushedTime := mgr.openedAt.Truncate(windowSize) + if waitTillFlushedTime.Equal(mgr.openedAt) { + waitTillFlushedTime = waitTillFlushedTime.Add(-windowSize) + } + + for numForwardedTimes, lastFlushedNanos := range fbr.ByNumForwardedTimes { + if lastFlushedNanos == 0 { + mgr.logger.Info("Encountered a window that was never flushed by leader", + zap.Time("now", now), + zap.Stringer("windowSize", windowSize), + zap.Stringer("flusherType", forwardedMetricListType), + zap.Int("shardID", int(shardID)), + zap.Int32("numForwardedTimes", numForwardedTimes)) + mgr.metrics.forwarded.windowNeverFlushed.Inc(1) + + if mgr.canLeadNotFlushed(now, windowSize) { + continue + } else { + return false + } + } + + if lastFlushedNanos <= waitTillFlushedTime.UnixNano() { + mgr.metrics.forwarded.flushWindowsNotEnded.Inc(1) + return false + } + } + } + + return true +} + +// canLeadNotFlushed determines whether the follower can takeover leadership +// for the window that was not (yet) flushed by leader. +// This is case is possible when leader encounters a metric at some resolution +// window for the first time in the shard it owns. +func (mgr *followerFlushManager) canLeadNotFlushed(now time.Time, windowSize time.Duration) bool { + windowStartAt := now.Truncate(windowSize) + return mgr.openedAt.Before(windowStartAt) +} + func (mgr *followerFlushManager) Close() { mgr.Wait() } func (mgr *followerFlushManager) flushersFromKVUpdateWithLock(buckets []*flushBucket) []flushersGroup { diff --git a/src/aggregator/aggregator/follower_flush_mgr_test.go b/src/aggregator/aggregator/follower_flush_mgr_test.go index 99e1192375..265bf4a004 100644 --- a/src/aggregator/aggregator/follower_flush_mgr_test.go +++ b/src/aggregator/aggregator/follower_flush_mgr_test.go @@ -24,7 +24,9 @@ import ( "testing" "time" + schema "github.com/m3db/m3/src/aggregator/generated/proto/flush" "github.com/m3db/m3/src/cluster/kv/mem" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/watch" "github.com/golang/mock/gomock" @@ -131,6 +133,113 @@ func TestFollowerFlushManagerCanNotLeadForwardedFlushWindowsNotEnded(t *testing. require.False(t, mgr.CanLead()) } +func TestFollowerFlushManagerCanLeadNotFlushed(t *testing.T) { + now := time.Unix(24*60*60, 0) + window10m := 10 * time.Minute + window1m := time.Minute + testFlushTimes := &schema.ShardSetFlushTimes{ + ByShard: map[uint32]*schema.ShardFlushTimes{ + 123: &schema.ShardFlushTimes{ + StandardByResolution: map[int64]int64{ + window10m.Nanoseconds(): 0, + }, + ForwardedByResolution: map[int64]*schema.ForwardedFlushTimesForResolution{ + window10m.Nanoseconds(): &schema.ForwardedFlushTimesForResolution{ + ByNumForwardedTimes: map[int32]int64{ + 1: 0, + }, + }, + }, + }, + }, + } + + runTestFn := func( + t *testing.T, + flushTimes *schema.ShardSetFlushTimes, + followerOpenedAt time.Time, + expectedCanLead bool, + ) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + doneCh := make(chan struct{}) + electionManager := NewMockElectionManager(ctrl) + electionManager.EXPECT().IsCampaigning().Return(true).AnyTimes() + clockOpts := clock.NewOptions().SetNowFn(func() time.Time { + return now + }) + opts := NewFlushManagerOptions(). + SetElectionManager(electionManager). + SetClockOptions(clockOpts) + mgr := newFollowerFlushManager(doneCh, opts).(*followerFlushManager) + + mgr.processed = flushTimes + mgr.openedAt = followerOpenedAt + require.Equal(t, expectedCanLead, mgr.CanLead()) + } + + t.Run("opened_on_the_window_start", func(t *testing.T) { + followerOpenedAt := now.Truncate(window10m) + expectedCanLead := false + runTestFn(t, testFlushTimes, followerOpenedAt, expectedCanLead) + }) + + t.Run("opened_after_the_window_start", func(t *testing.T) { + followerOpenedAt := now.Truncate(window10m).Add(1 * time.Second) + expectedCanLead := false + runTestFn(t, testFlushTimes, followerOpenedAt, expectedCanLead) + }) + + t.Run("opened_before_the_window_start", func(t *testing.T) { + followerOpenedAt := now.Truncate(window10m).Add(-1 * time.Second) + expectedCanLead := true + runTestFn(t, testFlushTimes, followerOpenedAt, expectedCanLead) + }) + + t.Run("standard_flushed_ok_and_unflushed_bad", func(t *testing.T) { + flushedAndUnflushedTimes := &schema.ShardSetFlushTimes{ + ByShard: map[uint32]*schema.ShardFlushTimes{ + 123: &schema.ShardFlushTimes{ + StandardByResolution: map[int64]int64{ + window1m.Nanoseconds(): now.Add(1 * time.Second).UnixNano(), + window10m.Nanoseconds(): 0, + }, + }, + }, + } + + followerOpenedAt := now + expectedCanLead := false + runTestFn(t, flushedAndUnflushedTimes, followerOpenedAt, expectedCanLead) + }) + + t.Run("forwarded_flushed_ok_and_unflushed_bad", func(t *testing.T) { + flushedAndUnflushedTimes := &schema.ShardSetFlushTimes{ + ByShard: map[uint32]*schema.ShardFlushTimes{ + 123: &schema.ShardFlushTimes{ + ForwardedByResolution: map[int64]*schema.ForwardedFlushTimesForResolution{ + window1m.Nanoseconds(): &schema.ForwardedFlushTimesForResolution{ + ByNumForwardedTimes: map[int32]int64{ + 1: now.Truncate(window1m).Add(1 * time.Second).UnixNano(), + }, + }, + window10m.Nanoseconds(): &schema.ForwardedFlushTimesForResolution{ + ByNumForwardedTimes: map[int32]int64{ + 1: 0, + }, + }, + }, + }, + }, + } + + followerOpenedAt := now + expectedCanLead := false + runTestFn(t, flushedAndUnflushedTimes, followerOpenedAt, expectedCanLead) + }) +} + func TestFollowerFlushManagerCanLeadNoTombstonedShards(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() From 81b8d826347a7e532b7283f20b9fea5f8fca90b9 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Sun, 1 Nov 2020 08:55:44 -0500 Subject: [PATCH 15/47] [read_index_segments] Add log that outlines which segment being processed (#2825) --- src/cmd/tools/read_index_segments/main/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cmd/tools/read_index_segments/main/main.go b/src/cmd/tools/read_index_segments/main/main.go index e314c76919..680e8019cc 100644 --- a/src/cmd/tools/read_index_segments/main/main.go +++ b/src/cmd/tools/read_index_segments/main/main.go @@ -126,6 +126,12 @@ func readNamespaceSegments( continue } + log.Info("reading block segments", + zap.String("namespace", nsID.String()), + zap.String("blockStart", infoFile.ID.BlockStart.String()), + zap.Int64("blockStartUnixNano", infoFile.ID.BlockStart.UnixNano()), + zap.Strings("files", infoFile.AbsoluteFilePaths)) + segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ ReaderOptions: fs.IndexReaderOpenOptions{ Identifier: infoFile.ID, From 1c7bb7ed9806e8a311fed69d1bf8ad93b7085ace Mon Sep 17 00:00:00 2001 From: teddywahle <69990143+teddywahle@users.noreply.github.com> Date: Sun, 1 Nov 2020 18:18:30 -0500 Subject: [PATCH 16/47] [query] Refactor groupByNodes and implement `nodes` arg in asPercent (#2816) --- .../graphite/native/aggregation_functions.go | 91 +++++++++++-------- .../native/aggregation_functions_test.go | 13 ++- .../graphite/native/builtin_functions.go | 55 +++++++++-- .../graphite/native/builtin_functions_test.go | 40 ++++++++ 4 files changed, 148 insertions(+), 51 deletions(-) diff --git a/src/query/graphite/native/aggregation_functions.go b/src/query/graphite/native/aggregation_functions.go index b4b82b0634..a0cd8e569c 100644 --- a/src/query/graphite/native/aggregation_functions.go +++ b/src/query/graphite/native/aggregation_functions.go @@ -105,8 +105,7 @@ func medianSeries(ctx *common.Context, series multiplePathSpecs) (ts.SeriesList, return ts.NewSeriesList(), err } numSteps := ts.NumSteps(start, end, millisPerStep) - values := ts.NewValues(ctx, millisPerStep, numSteps) - + values := ts.NewValues(ctx, millisPerStep, numSteps) valuesAtTime := make([]float64, len(normalized.Values)) for i := 0; i < numSteps; i++ { @@ -561,24 +560,59 @@ func applyByNode(ctx *common.Context, seriesList singlePathSpec, nodeNum int, te // // sumSeries(foo.by-function.server1.*.cpu.load5),sumSeries(foo.by-function.server2.*.cpu.load5),... func groupByNode(ctx *common.Context, series singlePathSpec, node int, fname string) (ts.SeriesList, error) { - metaSeries := make(map[string][]*ts.Series) - for _, s := range series.Values { - parts := strings.Split(s.Name(), ".") + return groupByNodes(ctx, series, fname, []int{node}...) +} + +func findFirstMetricExpression(seriesName string) (string, bool) { + idxOfRightParen := strings.Index(seriesName, ")") + if idxOfRightParen == -1 { + return "", false + } + substring := seriesName[:idxOfRightParen] + idxOfLeftParen := strings.LastIndex(substring, "(") + if idxOfLeftParen == -1 { + return "", false + } + return seriesName[idxOfLeftParen+1 : idxOfRightParen], true +} - n := node +func getParts(series *ts.Series) []string { + seriesName := series.Name() + if metricExpr, ok := findFirstMetricExpression(seriesName); ok { + seriesName = metricExpr + } + return strings.Split(seriesName, ".") +} + +func getAggregationKey(series *ts.Series, nodes []int) string { + parts := getParts(series) + + keys := make([]string, 0, len(nodes)) + for _, n := range nodes { if n < 0 { n = len(parts) + n } - if n >= len(parts) || n < 0 { - return aggregate(ctx, series, fname) + if n < len(parts) { + keys = append(keys, parts[n]) + } else { + keys = append(keys, "") } + } + return strings.Join(keys, ".") +} + +func getMetaSeriesGrouping(seriesList singlePathSpec, nodes []int) map[string][]*ts.Series { + metaSeries := make(map[string][]*ts.Series) - key := parts[n] - metaSeries[key] = append(metaSeries[key], s) + if len(nodes) > 0 { + for _, s := range seriesList.Values { + key := getAggregationKey(s, nodes) + metaSeries[key] = append(metaSeries[key], s) + } } - return applyFnToMetaSeries(ctx, series, metaSeries, fname) + return metaSeries } // Takes a serieslist and maps a callback to subgroups within as defined by multiple nodes @@ -592,37 +626,16 @@ func groupByNode(ctx *common.Context, series singlePathSpec, node int, fname str // sumSeries(ganglia.server2.*.cpu.load5),sumSeries(ganglia.server2.*.cpu.load10),sumSeries(ganglia.server2.*.cpu.load15),... // // NOTE: if len(nodes) = 0, aggregate all series into 1 series. -func groupByNodes(ctx *common.Context, series singlePathSpec, fname string, nodes ...int) (ts.SeriesList, error) { - metaSeries := make(map[string][]*ts.Series) - - nodeLen := len(nodes) - if nodeLen == 0 { - key := "*" // put into single group, not ideal, but more graphite-ish. - for _, s := range series.Values { - metaSeries[key] = append(metaSeries[key], s) - } - } else { - for _, s := range series.Values { - parts := strings.Split(s.Name(), ".") +func groupByNodes(ctx *common.Context, seriesList singlePathSpec, fname string, nodes ...int) (ts.SeriesList, error) { + metaSeries := getMetaSeriesGrouping(seriesList, nodes) - var keys []string - for _, n := range nodes { - if n < 0 { - n = len(parts) + n - } - - if n >= len(parts) || n < 0 { - return aggregate(ctx, series, fname) - } - - keys = append(keys, parts[n]) - } - key := strings.Join(keys, ".") - metaSeries[key] = append(metaSeries[key], s) - } + if len(metaSeries) == 0 { + // if nodes is an empty slice or every node in nodes exceeds the number + // of parts in each series, just treat it like aggregate + return aggregate(ctx, seriesList, fname) } - return applyFnToMetaSeries(ctx, series, metaSeries, fname) + return applyFnToMetaSeries(ctx, seriesList, metaSeries, fname) } func applyFnToMetaSeries(ctx *common.Context, series singlePathSpec, metaSeries map[string][]*ts.Series, fname string) (ts.SeriesList, error) { diff --git a/src/query/graphite/native/aggregation_functions_test.go b/src/query/graphite/native/aggregation_functions_test.go index 45d7054487..cfdd9d1cd5 100644 --- a/src/query/graphite/native/aggregation_functions_test.go +++ b/src/query/graphite/native/aggregation_functions_test.go @@ -712,7 +712,7 @@ func TestGroupByNodes(t *testing.T) { end, _ = time.Parse(time.RFC1123, "Mon, 27 Jul 2015 19:43:19 GMT") ctx = common.NewContext(common.ContextOptions{Start: start, End: end}) inputs = []*ts.Series{ - ts.NewSeries(ctx, "servers.foo-1.pod1.status.500", start, + ts.NewSeries(ctx, "transformNull(servers.foo-1.pod1.status.500)", start, ts.NewConstantValues(ctx, 2, 12, 10000)), ts.NewSeries(ctx, "servers.foo-2.pod1.status.500", start, ts.NewConstantValues(ctx, 4, 12, 10000)), @@ -755,6 +755,12 @@ func TestGroupByNodes(t *testing.T) { {"pod2.400", 40 * 12}, {"pod2.500", 10 * 12}, }}, + {"max", []int{2, 4, 100}, []result{ // test with a node number that exceeds num parts + {"pod1.400.", 30 * 12}, + {"pod1.500.", 6 * 12}, + {"pod2.400.", 40 * 12}, + {"pod2.500.", 10 * 12}, + }}, {"min", []int{2, -1}, []result{ // test negative index handling {"pod1.400", 20 * 12}, {"pod1.500", 2 * 12}, @@ -762,7 +768,10 @@ func TestGroupByNodes(t *testing.T) { {"pod2.500", 8 * 12}, }}, {"sum", []int{}, []result{ // test empty slice handing. - {"*", (2 + 4 + 6 + 8 + 10 + 20 + 30 + 40) * 12}, + {"sumSeries(transformNull(servers.foo-1.pod1.status.500),servers.foo-2.pod1.status.500,servers.foo-3.pod1.status.500,servers.foo-1.pod2.status.500,servers.foo-2.pod2.status.500,servers.foo-1.pod1.status.400,servers.foo-2.pod1.status.400,servers.foo-3.pod2.status.400)", (2 + 4 + 6 + 8 + 10 + 20 + 30 + 40) * 12}, + }}, + {"sum", []int{100}, []result{ // test all nodes out of bounds + {"", (2 + 4 + 6 + 8 + 10 + 20 + 30 + 40) * 12}, }}, } diff --git a/src/query/graphite/native/builtin_functions.go b/src/query/graphite/native/builtin_functions.go index 79fa1e493f..4528c07c1d 100644 --- a/src/query/graphite/native/builtin_functions.go +++ b/src/query/graphite/native/builtin_functions.go @@ -990,7 +990,7 @@ func exponentialMovingAverage(ctx *common.Context, input singlePathSpec, windowS } // totalFunc takes an index and returns a total value for that index -type totalFunc func(int) float64 +type totalFunc func(int, *ts.Series) float64 func totalBySum(seriesList []*ts.Series, index int) float64 { s, hasValue := 0.0, false @@ -1008,7 +1008,7 @@ func totalBySum(seriesList []*ts.Series, index int) float64 { } // asPercent calculates a percentage of the total of a wildcard series. -func asPercent(ctx *common.Context, input singlePathSpec, total genericInterface) (ts.SeriesList, error) { +func asPercent(ctx *common.Context, input singlePathSpec, total genericInterface, nodes ...int) (ts.SeriesList, error) { if len(input.Values) == 0 { return ts.SeriesList(input), nil } @@ -1029,24 +1029,56 @@ func asPercent(ctx *common.Context, input singlePathSpec, total genericInterface if total.Len() == 0 { // normalize input and sum up input as the total series toNormalize = input.Values - tf = func(idx int) float64 { return totalBySum(normalized, idx) } + tf = func(idx int, _ *ts.Series) float64 { return totalBySum(normalized, idx) } } else { // check total is a single-series list and normalize all of them if total.Len() != 1 { err := errors.NewInvalidParamsError(errors.New("total must be a single series")) return ts.NewSeriesList(), err } + if len(nodes) > 0 { + // group the series by specified nodes and then sum those groups + groupedTotal, err := groupByNodes(ctx, input, "sum", nodes...) + if err != nil { + return ts.NewSeriesList(), err + } + toNormalize = append(input.Values, groupedTotal.Values[0]) + metaSeriesSumByKey := make(map[string]*ts.Series) - toNormalize = append(input.Values, total.Values[0]) - tf = func(idx int) float64 { return normalized[len(normalized)-1].ValueAt(idx) } - totalText = total.Values[0].Name() + // map the aggregation key to the aggregated series + for _, series := range groupedTotal.Values { + metaSeriesSumByKey[series.Name()] = series + } + + tf = func(idx int, series *ts.Series) float64 { + // find which aggregation key this series falls under + // and return the sum for that aggregated group + key := getAggregationKey(series, nodes) + return metaSeriesSumByKey[key].ValueAt(idx) + } + totalText = groupedTotal.Values[0].Name() + } else { + toNormalize = append(input.Values, total.Values[0]) + tf = func(idx int, _ *ts.Series) float64 { return normalized[len(normalized)-1].ValueAt(idx) } + totalText = total.Values[0].Name() + } } case float64: toNormalize = input.Values - tf = func(idx int) float64 { return totalArg } + tf = func(idx int, _ *ts.Series) float64 { return totalArg } totalText = fmt.Sprintf(common.FloatingPointFormat, totalArg) + case nil: + // if total is nil, the total is the sum of all the input series + toNormalize = input.Values + var err error + summedSeries, err := sumSeries(ctx, multiplePathSpecs(input)) + if err != nil { + return ts.NewSeriesList(), err + } + tf = func(idx int, _ *ts.Series) float64 { return summedSeries.Values[0].ValueAt(idx) } + totalText = summedSeries.Values[0].Name() default: - err := errors.NewInvalidParamsError(errors.New("total is neither an int nor a series")) + err := errors.NewInvalidParamsError(errors.New("total must be either an int, a series, or nil")) return ts.NewSeriesList(), err } @@ -1066,8 +1098,8 @@ func asPercent(ctx *common.Context, input singlePathSpec, total genericInterface values = append(values, percents) } for i := 0; i < normalized[0].Len(); i++ { - t := tf(i) for j := 0; j < numInputSeries; j++ { + t := tf(i, normalized[j]) v := normalized[j].ValueAt(i) if !math.IsNaN(v) && !math.IsNaN(t) && t != 0 { values[j].SetValueAt(i, 100.0*v/t) @@ -2348,6 +2380,7 @@ func init() { }) MustRegisterFunction(asPercent).WithDefaultParams(map[uint8]interface{}{ 2: []*ts.Series(nil), // total + 3: nil, // nodes }) MustRegisterFunction(averageAbove) MustRegisterFunction(averageSeries) @@ -2378,7 +2411,9 @@ func init() { MustRegisterFunction(groupByNode).WithDefaultParams(map[uint8]interface{}{ 3: "average", // fname }) - MustRegisterFunction(groupByNodes) + MustRegisterFunction(groupByNodes).WithDefaultParams(map[uint8]interface{}{ + 3: nil, // nodes + }) MustRegisterFunction(highest).WithDefaultParams(map[uint8]interface{}{ 2: 1, // n, 3: "average", // f diff --git a/src/query/graphite/native/builtin_functions_test.go b/src/query/graphite/native/builtin_functions_test.go index 23855073d6..68e0108b42 100644 --- a/src/query/graphite/native/builtin_functions_test.go +++ b/src/query/graphite/native/builtin_functions_test.go @@ -1966,6 +1966,46 @@ func TestAsPercentWithFloatTotal(t *testing.T) { } } +func TestAsPercentWithNilTotal(t *testing.T) { + ctx := common.NewTestContext() + defer ctx.Close() + + nan := math.NaN() + tests := []struct { + valuesStep int + values []float64 + outputStep int + output []float64 + }{ + { + 60, + []float64{12.0, 14.0, 16.0, nan, 20.0}, + 60, + []float64{100, 100, 100, nan, 100}, + }, + } + + for _, test := range tests { + timeSeries := ts.NewSeries(ctx, "", ctx.StartTime, + common.NewTestSeriesValues(ctx, test.valuesStep, test.values)) + r, err := asPercent(ctx, singlePathSpec{ + Values: []*ts.Series{timeSeries}, + }, nil) + require.NoError(t, err) + + output := r.Values + require.Equal(t, 1, len(output)) + require.Equal(t, output[0].MillisPerStep(), test.outputStep) + expectedName := fmt.Sprintf("asPercent(, sumSeries())") + assert.Equal(t, expectedName, output[0].Name()) + + for step := 0; step < output[0].Len(); step++ { + v := output[0].ValueAt(step) + xtest.Equalish(t, math.Trunc(v), test.output[step]) + } + } +} + func TestAsPercentWithSeriesList(t *testing.T) { ctx := common.NewTestContext() defer ctx.Close() From 83fb11fa71dfd35e3642f85d2337b908ac31d2a5 Mon Sep 17 00:00:00 2001 From: Gediminas Guoba Date: Tue, 3 Nov 2020 11:06:56 +0200 Subject: [PATCH 17/47] [large-tiles] Additional metrics added (#2720) --- src/dbnode/integration/large_tiles_test.go | 12 ++++++--- .../server/tchannelthrift/node/service.go | 12 ++++++--- .../tchannelthrift/node/service_test.go | 11 ++++++-- src/dbnode/storage/database.go | 25 ++++++++++++++++--- src/dbnode/storage/database_test.go | 16 +++++++----- src/dbnode/storage/namespace.go | 2 ++ src/dbnode/storage/namespace_test.go | 6 ++++- src/dbnode/storage/types.go | 3 ++- 8 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src/dbnode/integration/large_tiles_test.go b/src/dbnode/integration/large_tiles_test.go index d4d915375d..b8a32739ec 100644 --- a/src/dbnode/integration/large_tiles_test.go +++ b/src/dbnode/integration/large_tiles_test.go @@ -110,7 +110,11 @@ func TestReadAggregateWrite(t *testing.T) { require.True(t, flushed) log.Info("verified data has been cold flushed", zap.Duration("took", time.Since(start))) - aggOpts, err := storage.NewAggregateTilesOptions(dpTimeStart, dpTimeStart.Add(blockSizeT), time.Hour) + aggOpts, err := storage.NewAggregateTilesOptions( + dpTimeStart, dpTimeStart.Add(blockSizeT), time.Hour, + trgNs.ID(), + storageOpts.InstrumentOptions(), + ) require.NoError(t, err) log.Info("Starting aggregation") @@ -204,9 +208,9 @@ func setupServer(t *testing.T) (TestSetup, namespace.Metadata, namespace.Metadat idxOpts = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSize) idxOptsT = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSizeT) nsOpts = namespace.NewOptions(). - SetRetentionOptions(rOpts). - SetIndexOptions(idxOpts). - SetColdWritesEnabled(true) + SetRetentionOptions(rOpts). + SetIndexOptions(idxOpts). + SetColdWritesEnabled(true) nsOptsT = namespace.NewOptions(). SetRetentionOptions(rOptsT). SetIndexOptions(idxOptsT) diff --git a/src/dbnode/network/server/tchannelthrift/node/service.go b/src/dbnode/network/server/tchannelthrift/node/service.go index 6bc01f9ce4..19a973c317 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service.go +++ b/src/dbnode/network/server/tchannelthrift/node/service.go @@ -585,14 +585,18 @@ func (s *service) aggregateTiles( if err != nil { return 0, tterrors.NewBadRequestError(err) } - opts, err := storage.NewAggregateTilesOptions(start, end, step) - if err != nil { - return 0, tterrors.NewBadRequestError(err) - } sourceNsID := s.pools.id.GetStringID(ctx, req.SourceNamespace) targetNsID := s.pools.id.GetStringID(ctx, req.TargetNamespace) + opts, err := storage.NewAggregateTilesOptions( + start, end, step, + sourceNsID, + s.opts.InstrumentOptions()) + if err != nil { + return 0, tterrors.NewBadRequestError(err) + } + processedTileCount, err := db.AggregateTiles(ctx, sourceNsID, targetNsID, opts) if err != nil { return processedTileCount, convert.ToRPCError(err) diff --git a/src/dbnode/network/server/tchannelthrift/node/service_test.go b/src/dbnode/network/server/tchannelthrift/node/service_test.go index de418a5f3c..0f376001b1 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service_test.go +++ b/src/dbnode/network/server/tchannelthrift/node/service_test.go @@ -3138,8 +3138,15 @@ func TestServiceAggregateTiles(t *testing.T) { ctx, ident.NewIDMatcher(sourceNsID), ident.NewIDMatcher(targetNsID), - storage.AggregateTilesOptions{Start: start, End: end, Step: stepDuration}, - ).Return(int64(4), nil) + gomock.Any(), + ).DoAndReturn(func(gotCtx, gotSourceNsID, gotTargetNsID interface{}, opts storage.AggregateTilesOptions) (int64, error) { + require.NotNil(t, opts) + require.Equal(t, start, opts.Start) + require.Equal(t, end, opts.End) + require.Equal(t, stepDuration, opts.Step) + require.NotNil(t, opts.InsOptions) + return int64(4), nil + }) result, err := service.AggregateTiles(tctx, &rpc.AggregateTilesRequest{ SourceNamespace: sourceNsID, diff --git a/src/dbnode/storage/database.go b/src/dbnode/storage/database.go index d00c47787b..495387fe44 100644 --- a/src/dbnode/storage/database.go +++ b/src/dbnode/storage/database.go @@ -44,6 +44,7 @@ import ( "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" + "github.com/m3db/m3/src/x/instrument" xopentracing "github.com/m3db/m3/src/x/opentracing" xtime "github.com/m3db/m3/src/x/time" @@ -76,6 +77,7 @@ var ( // errWriterDoesNotImplementWriteBatch is raised when the provided ts.BatchWriter does not implement // ts.WriteBatch. errWriterDoesNotImplementWriteBatch = errors.New("provided writer does not implement ts.WriteBatch") + aggregationsInProgress int32 ) type databaseState int @@ -1377,6 +1379,14 @@ func (d *db) AggregateTiles( targetNsID ident.ID, opts AggregateTilesOptions, ) (int64, error) { + jobInProgress := opts.InsOptions.MetricsScope().Gauge("aggregations-in-progress") + atomic.AddInt32(&aggregationsInProgress, 1) + jobInProgress.Update(float64(aggregationsInProgress)) + defer func() { + atomic.AddInt32(&aggregationsInProgress, -1) + jobInProgress.Update(float64(aggregationsInProgress)) + }() + ctx, sp, sampled := ctx.StartSampledTraceSpan(tracepoint.DBAggregateTiles) if sampled { sp.LogFields( @@ -1408,6 +1418,9 @@ func (d *db) AggregateTiles( zap.String("targetNs", targetNsID.String()), zap.Error(err), ) + opts.InsOptions.MetricsScope().Counter("aggregation.errors").Inc(1) + } else { + opts.InsOptions.MetricsScope().Counter("aggregation.success").Inc(1) } return processedTileCount, err } @@ -1460,6 +1473,8 @@ func (m metadatas) String() (string, error) { func NewAggregateTilesOptions( start, end time.Time, step time.Duration, + targetNsID ident.ID, + insOpts instrument.Options, ) (AggregateTilesOptions, error) { if !end.After(start) { return AggregateTilesOptions{}, fmt.Errorf("AggregateTilesOptions.End must be after Start, got %s - %s", start, end) @@ -1469,9 +1484,13 @@ func NewAggregateTilesOptions( return AggregateTilesOptions{}, fmt.Errorf("AggregateTilesOptions.Step must be positive, got %s", step) } + scope := insOpts.MetricsScope().SubScope("computed-namespace") + insOpts = insOpts.SetMetricsScope(scope.Tagged(map[string]string{"target-namespace": targetNsID.String()})) + return AggregateTilesOptions{ - Start: start, - End: end, - Step: step, + Start: start, + End: end, + Step: step, + InsOptions: insOpts, }, nil } diff --git a/src/dbnode/storage/database_test.go b/src/dbnode/storage/database_test.go index fa71db5446..b4406a7e72 100644 --- a/src/dbnode/storage/database_test.go +++ b/src/dbnode/storage/database_test.go @@ -51,6 +51,7 @@ import ( "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" + "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" "github.com/m3db/m3/src/x/serialize" xtest "github.com/m3db/m3/src/x/test" @@ -1519,8 +1520,9 @@ func TestDatabaseAggregateTiles(t *testing.T) { start = time.Now().Truncate(time.Hour) ) - opts, err := NewAggregateTilesOptions(start, start.Add(-time.Second), time.Minute) + opts, err := NewAggregateTilesOptions(start, start.Add(-time.Second), time.Minute, targetNsID, d.opts.InstrumentOptions()) require.Error(t, err) + opts.InsOptions = d.opts.InstrumentOptions() sourceNs := dbAddNewMockNamespace(ctrl, d, sourceNsID.String()) targetNs := dbAddNewMockNamespace(ctrl, d, targetNsID.String()) @@ -1533,19 +1535,21 @@ func TestDatabaseAggregateTiles(t *testing.T) { func TestNewAggregateTilesOptions(t *testing.T) { start := time.Now().Truncate(time.Hour) + targetNs := ident.StringID("target") + insOpts := instrument.NewOptions() - _, err := NewAggregateTilesOptions(start, start.Add(-time.Second), time.Minute) + _, err := NewAggregateTilesOptions(start, start.Add(-time.Second), time.Minute, targetNs, insOpts) assert.Error(t, err) - _, err = NewAggregateTilesOptions(start, start, time.Minute) + _, err = NewAggregateTilesOptions(start, start, time.Minute, targetNs, insOpts) assert.Error(t, err) - _, err = NewAggregateTilesOptions(start, start.Add(time.Second), -time.Minute) + _, err = NewAggregateTilesOptions(start, start.Add(time.Second), -time.Minute, targetNs, insOpts) assert.Error(t, err) - _, err = NewAggregateTilesOptions(start, start.Add(time.Second), 0) + _, err = NewAggregateTilesOptions(start, start.Add(time.Second), 0, targetNs, insOpts) assert.Error(t, err) - _, err = NewAggregateTilesOptions(start, start.Add(time.Second), time.Minute) + _, err = NewAggregateTilesOptions(start, start.Add(time.Second), time.Minute, targetNs, insOpts) assert.NoError(t, err) } diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index 31b8148a1d..6465df365d 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -1752,6 +1752,7 @@ func (n *dbNamespace) aggregateTiles( n.RUnlock() var ( + processedShards = opts.InsOptions.MetricsScope().Counter("processed-shards") targetShards = n.OwnedShards() bytesPool = sourceNs.StorageOptions().BytesPool() fsOptions = sourceNs.StorageOptions().CommitLogOptions().FilesystemOptions() @@ -1795,6 +1796,7 @@ func (n *dbNamespace) aggregateTiles( sourceNs.ID(), sourceShard.ID(), blockReaders, writer, sourceBlockVolumes, opts, nsCtx.Schema) processedTileCount += shardProcessedTileCount + processedShards.Inc(1) if err != nil { return 0, fmt.Errorf("shard %d aggregation failed: %v", targetShard.ID(), err) } diff --git a/src/dbnode/storage/namespace_test.go b/src/dbnode/storage/namespace_test.go index dbff29da6c..978ca641bb 100644 --- a/src/dbnode/storage/namespace_test.go +++ b/src/dbnode/storage/namespace_test.go @@ -44,6 +44,7 @@ import ( "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" + "github.com/m3db/m3/src/x/instrument" xtest "github.com/m3db/m3/src/x/test" xtime "github.com/m3db/m3/src/x/time" @@ -1413,12 +1414,15 @@ func TestNamespaceAggregateTiles(t *testing.T) { sourceBlockSize = time.Hour targetBlockSize = 2 * time.Hour start = time.Now().Truncate(targetBlockSize) - opts = AggregateTilesOptions{Start: start, End: start.Add(targetBlockSize)} secondSourceBlockStart = start.Add(sourceBlockSize) sourceShard0ID uint32 = 10 sourceShard1ID uint32 = 20 + insOpts = instrument.NewOptions() ) + opts, err := NewAggregateTilesOptions(start, start.Add(targetBlockSize), time.Second, targetNsID, insOpts) + require.NoError(t, err) + sourceNs, sourceCloser := newTestNamespaceWithIDOpts(t, sourceNsID, namespace.NewOptions()) defer sourceCloser() sourceNs.bootstrapState = Bootstrapped diff --git a/src/dbnode/storage/types.go b/src/dbnode/storage/types.go index d9aa6cb52a..b7753bc9f8 100644 --- a/src/dbnode/storage/types.go +++ b/src/dbnode/storage/types.go @@ -1390,7 +1390,8 @@ type AggregateTilesOptions struct { // Start and End specify the aggregation window. Start, End time.Time // Step is the downsampling step. - Step time.Duration + Step time.Duration + InsOptions instrument.Options } // NamespaceHooks allows dynamic plugging into the namespace lifecycle. From c9ad6676b154798ab4bc187c15d9cd3cd1301482 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 3 Nov 2020 20:58:39 +1100 Subject: [PATCH 18/47] [query] Tests for when function argument is an expression (#2809) --- src/query/parser/promql/parse_test.go | 55 +++++++++++++++++++ .../compatibility/testdata/functions.test | 12 ++-- .../compatibility/testdata/operators.test | 18 +++--- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/query/parser/promql/parse_test.go b/src/query/parser/promql/parse_test.go index 37ec76a26d..656d578a05 100644 --- a/src/query/parser/promql/parse_test.go +++ b/src/query/parser/promql/parse_test.go @@ -541,6 +541,61 @@ func TestMissingTagsDoNotPanic(t *testing.T) { assert.NotPanics(t, func() { _, _, _ = p.DAG() }) } +var functionArgumentExpressionTests = []struct { + name string + q string +}{ + { + "scalar argument", + "vector(((1)))", + }, + { + "string argument", + `label_join(up, ("foo"), ((",")), ((("bar"))))`, + }, + { + "vector argument", + "abs(((foo)))", + }, + { + "matrix argument", + "stddev_over_time(((metric[1m])))", + }, +} + +func TestExpressionsInFunctionArgumentsDoNotError(t *testing.T) { + for _, tt := range functionArgumentExpressionTests { + t.Run(tt.name, func(t *testing.T) { + p, err := Parse(tt.q, time.Second, models.NewTagOptions(), NewParseOptions()) + require.NoError(t, err) + _, _, err = p.DAG() + require.NoError(t, err) + }) + } +} + +var invalidFunctionArgumentsTests = []string{ + "vector(())", + "vector((1)", + "vector(metric)", + `label_join(up, "f" + "oo", ",", "ba" + "r")`, + `label_join(up, 1, ",", 2)`, + `label_join("up", "foo", ",", "bar")`, + "abs(1)", + "abs(())", + "stddev_over_time(metric[1m]+1)", + "stddev_over_time(metric)", +} + +func TestParseInvalidFunctionArgumentsErrors(t *testing.T) { + for _, q := range invalidFunctionArgumentsTests { + t.Run(q, func(t *testing.T) { + _, err := Parse(q, time.Second, models.NewTagOptions(), NewParseOptions()) + require.Error(t, err) + }) + } +} + func TestCustomParseOptions(t *testing.T) { q := "query" v := "foo" diff --git a/src/query/test/compatibility/testdata/functions.test b/src/query/test/compatibility/testdata/functions.test index bef714ed38..e311bdc461 100644 --- a/src/query/test/compatibility/testdata/functions.test +++ b/src/query/test/compatibility/testdata/functions.test @@ -414,8 +414,8 @@ eval instant at 1m stdvar_over_time(metric[1m]) eval instant at 1m stddev_over_time(metric[1m]) {} 3.249615 -# FAILING issue #19. eval instant at 1m stddev_over_time((metric[1m])) -# {} 3.249615 +eval instant at 1m stddev_over_time((metric[1m])) + {} 3.249615 # Tests for stddev_over_time and stdvar_over_time #4927. clear @@ -471,10 +471,10 @@ eval instant at 1m quantile_over_time(2, data[1m]) {test="three samples"} +Inf {test="uneven samples"} +Inf -# FAILING issue #19. eval instant at 1m (quantile_over_time(2, (data[1m]))) -# {test="two samples"} +Inf -# {test="three samples"} +Inf -# {test="uneven samples"} +Inf +eval instant at 1m (quantile_over_time(2, (data[1m]))) + {test="two samples"} +Inf + {test="three samples"} +Inf + {test="uneven samples"} +Inf clear diff --git a/src/query/test/compatibility/testdata/operators.test b/src/query/test/compatibility/testdata/operators.test index 33137786c9..884df0dc55 100644 --- a/src/query/test/compatibility/testdata/operators.test +++ b/src/query/test/compatibility/testdata/operators.test @@ -115,15 +115,15 @@ eval instant at 50m rate(http_requests[25m]) * 25 * 60 {group="production", instance="1", job="api-server"} 100 {group="production", instance="1", job="app-server"} 300 -# FAILING issue #19. eval instant at 50m (rate((http_requests[25m])) * 25) * 60 -# {group="canary", instance="0", job="api-server"} 150 -# {group="canary", instance="0", job="app-server"} 350 -# {group="canary", instance="1", job="api-server"} 200 -# {group="canary", instance="1", job="app-server"} 400 -# {group="production", instance="0", job="api-server"} 50 -# {group="production", instance="0", job="app-server"} 249.99999999999997 -# {group="production", instance="1", job="api-server"} 100 -# {group="production", instance="1", job="app-server"} 300 +eval instant at 50m (rate((http_requests[25m])) * 25) * 60 + {group="canary", instance="0", job="api-server"} 150 + {group="canary", instance="0", job="app-server"} 350 + {group="canary", instance="1", job="api-server"} 200 + {group="canary", instance="1", job="app-server"} 400 + {group="production", instance="0", job="api-server"} 50 + {group="production", instance="0", job="app-server"} 249.99999999999997 + {group="production", instance="1", job="api-server"} 100 + {group="production", instance="1", job="app-server"} 300 # FAILING order of rows differs. eval instant at 50m http_requests{group="canary"} and http_requests{instance="0"} From 39937ccc59a0defb1332831caf72fa13548ec5af Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Nov 2020 10:58:32 -0500 Subject: [PATCH 19/47] [query] Add non-ready namespaces to Clusters interface and use in /namespace/ready endpoint (#2828) --- src/dbnode/namespace/staging.go | 15 +++++++- src/query/api/v1/handler/namespace/ready.go | 8 +++- .../api/v1/handler/namespace/ready_test.go | 18 +++++---- src/query/storage/m3/cluster.go | 10 ++++- src/query/storage/m3/cluster_test.go | 4 ++ src/query/storage/m3/dynamic_cluster.go | 38 ++++++++++++------- src/query/storage/m3/dynamic_cluster_test.go | 17 ++++++--- 7 files changed, 80 insertions(+), 30 deletions(-) diff --git a/src/dbnode/namespace/staging.go b/src/dbnode/namespace/staging.go index cf593925af..c2f77399d4 100644 --- a/src/dbnode/namespace/staging.go +++ b/src/dbnode/namespace/staging.go @@ -33,12 +33,12 @@ type StagingState struct { } // Status returns the StagingStatus for a namespace. -func (s *StagingState) Status() StagingStatus { +func (s StagingState) Status() StagingStatus { return s.status } // Validate validates the StagingState object. -func (s *StagingState) Validate() error { +func (s StagingState) Validate() error { var validStatus bool for _, status := range validStagingStatuses { if status == s.Status() { @@ -65,3 +65,14 @@ func NewStagingState(status nsproto.StagingStatus) (StagingState, error) { } return StagingState{}, fmt.Errorf("invalid namespace status: %v", status) } + +func (s StagingStatus) String() string { + switch s { + case ReadyStagingStatus: + return "ready" + case InitializingStagingStatus: + return "initializing" + default: + return "unknown" + } +} diff --git a/src/query/api/v1/handler/namespace/ready.go b/src/query/api/v1/handler/namespace/ready.go index eefa19f9f5..65ef17819f 100644 --- a/src/query/api/v1/handler/namespace/ready.go +++ b/src/query/api/v1/handler/namespace/ready.go @@ -132,6 +132,12 @@ func (h *ReadyHandler) Ready(req *admin.NamespaceReadyRequest, opts handleroptio return false, xerrors.NewInvalidParamsError(fmt.Errorf("namespace %v not found", req.Name)) } + // Just return if namespace is already ready. + currentState := ns.Options().StagingState() + if currentState.Status() == namespace.ReadyStagingStatus { + return true, nil + } + // If we're not forcing the staging state, check db nodes to see if namespace is ready. if !req.Force { if err := h.checkDBNodes(req.Name); err != nil { @@ -185,7 +191,7 @@ func (h *ReadyHandler) checkDBNodes(namespace string) error { session client.Session id ident.ID ) - for _, clusterNamespace := range h.clusters.ClusterNamespaces() { + for _, clusterNamespace := range h.clusters.NonReadyClusterNamespaces() { if clusterNamespace.NamespaceID().String() == namespace { session = clusterNamespace.Session() id = clusterNamespace.NamespaceID() diff --git a/src/query/api/v1/handler/namespace/ready_test.go b/src/query/api/v1/handler/namespace/ready_test.go index 2e8fde6e2f..ed871bb6b3 100644 --- a/src/query/api/v1/handler/namespace/ready_test.go +++ b/src/query/api/v1/handler/namespace/ready_test.go @@ -57,7 +57,7 @@ func TestNamespaceReadyHandler(t *testing.T) { options: m3.ClusterNamespaceOptions{}, } testClusters := testClusters{ - namespaces: []m3.ClusterNamespace{&testClusterNs}, + nonReadyNamespaces: []m3.ClusterNamespace{&testClusterNs}, } mockSession.EXPECT().FetchTaggedIDs(testClusterNs.NamespaceID(), index.Query{Query: idx.NewAllQuery()}, index.QueryOptions{SeriesLimit: 1, DocsLimit: 1}).Return(nil, client.FetchResponseMetadata{}, nil) @@ -70,7 +70,7 @@ func TestNamespaceReadyHandler(t *testing.T) { "name": "testNamespace", } - req := httptest.NewRequest("POST", "/namespace/mark_ready", + req := httptest.NewRequest("POST", "/namespace/ready", xjson.MustNewTestReader(t, jsonInput)) require.NotNil(t, req) @@ -105,7 +105,7 @@ func TestNamespaceReadyHandlerWithForce(t *testing.T) { "force": true, } - req := httptest.NewRequest("POST", "/namespace/mark_ready", + req := httptest.NewRequest("POST", "/namespace/ready", xjson.MustNewTestReader(t, jsonInput)) require.NotNil(t, req) @@ -138,7 +138,7 @@ func TestNamespaceReadyFailIfNoClusters(t *testing.T) { "name": "testNamespace", } - req := httptest.NewRequest("POST", "/namespace/mark_ready", + req := httptest.NewRequest("POST", "/namespace/ready", xjson.MustNewTestReader(t, jsonInput)) require.NotNil(t, req) @@ -163,7 +163,7 @@ func TestNamespaceReadyFailIfNamespaceMissing(t *testing.T) { "name": "missingNamespace", } - req := httptest.NewRequest("POST", "/namespace/mark_ready", + req := httptest.NewRequest("POST", "/namespace/ready", xjson.MustNewTestReader(t, jsonInput)) require.NotNil(t, req) @@ -220,11 +220,15 @@ func (t *testClusterNamespace) Session() client.Session { } type testClusters struct { - namespaces m3.ClusterNamespaces + nonReadyNamespaces m3.ClusterNamespaces } func (t *testClusters) ClusterNamespaces() m3.ClusterNamespaces { - return t.namespaces + panic("implement me") +} + +func (t *testClusters) NonReadyClusterNamespaces() m3.ClusterNamespaces { + return t.nonReadyNamespaces } func (t *testClusters) Close() error { diff --git a/src/query/storage/m3/cluster.go b/src/query/storage/m3/cluster.go index e525b9f695..28d6ff3dc8 100644 --- a/src/query/storage/m3/cluster.go +++ b/src/query/storage/m3/cluster.go @@ -48,9 +48,12 @@ var ( type Clusters interface { io.Closer - // ClusterNamespaces returns all known cluster namespaces. + // ClusterNamespaces returns all known and ready cluster namespaces. ClusterNamespaces() ClusterNamespaces + // NonReadyClusterNamespaces returns all cluster namespaces not in the ready state. + NonReadyClusterNamespaces() ClusterNamespaces + // UnaggregatedClusterNamespace returns the valid unaggregated // cluster namespace. UnaggregatedClusterNamespace() ClusterNamespace @@ -250,6 +253,11 @@ func (c *clusters) ClusterNamespaces() ClusterNamespaces { return c.namespaces } +func (c *clusters) NonReadyClusterNamespaces() ClusterNamespaces { + // statically configured cluster namespaces are always considered ready. + return nil +} + func (c *clusters) UnaggregatedClusterNamespace() ClusterNamespace { return c.unaggregatedNamespace } diff --git a/src/query/storage/m3/cluster_test.go b/src/query/storage/m3/cluster_test.go index c0273038a3..b538ec0dc9 100644 --- a/src/query/storage/m3/cluster_test.go +++ b/src/query/storage/m3/cluster_test.go @@ -205,6 +205,10 @@ func (n *noopCluster) ClusterNamespaces() ClusterNamespaces { panic("implement me") } +func (n *noopCluster) NonReadyClusterNamespaces() ClusterNamespaces { + panic("implement me") +} + func (n *noopCluster) UnaggregatedClusterNamespace() ClusterNamespace { panic("implement me") } diff --git a/src/query/storage/m3/dynamic_cluster.go b/src/query/storage/m3/dynamic_cluster.go index e3607618ea..b68ccf18e1 100644 --- a/src/query/storage/m3/dynamic_cluster.go +++ b/src/query/storage/m3/dynamic_cluster.go @@ -51,6 +51,7 @@ type dynamicCluster struct { sync.RWMutex allNamespaces ClusterNamespaces + nonReadyNamespaces ClusterNamespaces unaggregatedNamespace ClusterNamespace aggregatedNamespaces map[RetentionResolution]ClusterNamespace namespacesByEtcdCluster map[int]clusterNamespaceLookup @@ -234,18 +235,6 @@ func (d *dynamicCluster) updateNamespacesByEtcdClusterWithLock( continue } - stagingState := newNsMd.Options().StagingState() - switch stagingState.Status() { - case namespace.UnknownStagingStatus: - d.logger.Error("namespace has unknown staging status", - zap.String("namespace", newNsMd.ID().String())) - continue - case namespace.InitializingStagingStatus: - d.logger.Debug("ignoring namespace while initializing", - zap.String("namespace", newNsMd.ID().String())) - continue - } - // Namespace has been added. newClusterNamespaces, err := toClusterNamespaces(clusterCfg, newNsMd) if err != nil { @@ -326,13 +315,27 @@ func (d *dynamicCluster) updateClusterNamespacesWithLock() { var ( newNamespaces = make(ClusterNamespaces, 0, nsCount) + newNonReadyNamespaces = make(ClusterNamespaces, 0, nsCount) newAggregatedNamespaces = make(map[RetentionResolution]ClusterNamespace) newUnaggregatedNamespace ClusterNamespace ) for _, nsMap := range d.namespacesByEtcdCluster { - for _, clusterNamespaces := range nsMap.metadataToClusterNamespaces { + for md, clusterNamespaces := range nsMap.metadataToClusterNamespaces { for _, clusterNamespace := range clusterNamespaces { + status := md.Options().StagingState().Status() + // Don't make non-ready namespaces available for read/write in coordinator, but track + // them so that we can have the DB session available when we need to check their + // readiness in the /namespace/ready check. + if status != namespace.ReadyStagingStatus { + d.logger.Info("namespace has non-ready staging state status", + zap.String("namespace", md.ID().String()), + zap.String("status", status.String())) + + newNonReadyNamespaces = append(newNonReadyNamespaces, clusterNamespace) + continue + } + attrs := clusterNamespace.Options().Attributes() if attrs.MetricsType == storagemetadata.UnaggregatedMetricsType { if newUnaggregatedNamespace != nil { @@ -371,6 +374,7 @@ func (d *dynamicCluster) updateClusterNamespacesWithLock() { d.unaggregatedNamespace = newUnaggregatedNamespace d.aggregatedNamespaces = newAggregatedNamespaces + d.nonReadyNamespaces = newNonReadyNamespaces d.allNamespaces = newNamespaces } @@ -406,6 +410,14 @@ func (d *dynamicCluster) ClusterNamespaces() ClusterNamespaces { return allNamespaces } +func (d *dynamicCluster) NonReadyClusterNamespaces() ClusterNamespaces { + d.RLock() + nonReadyNamespaces := d.nonReadyNamespaces + d.RUnlock() + + return nonReadyNamespaces +} + func (d *dynamicCluster) UnaggregatedClusterNamespace() ClusterNamespace { d.RLock() unaggregatedNamespaces := d.unaggregatedNamespace diff --git a/src/query/storage/m3/dynamic_cluster_test.go b/src/query/storage/m3/dynamic_cluster_test.go index db5710ab3c..e5f8d9a584 100644 --- a/src/query/storage/m3/dynamic_cluster_test.go +++ b/src/query/storage/m3/dynamic_cluster_test.go @@ -186,7 +186,7 @@ func TestDynamicClustersWithUpdates(t *testing.T) { downsample: &ClusterNamespaceDownsampleOptions{All: true}, }) - return found && assertClusterNamespaceIDs(clusters, []ident.ID{defaultTestNs1ID, defaultTestNs2ID}) + return found && assertClusterNamespaceIDs(clusters.ClusterNamespaces(), []ident.ID{defaultTestNs1ID, defaultTestNs2ID}) }, time.Second)) } @@ -286,7 +286,7 @@ func TestDynamicClustersWithMultipleInitializers(t *testing.T) { fooNsID, barNsID}) } -func TestDynamicClustersNotReadyNamespace(t *testing.T) { +func TestDynamicClustersNonReadyNamespace(t *testing.T) { ctrl := xtest.NewController(t) defer ctrl.Finish() @@ -325,6 +325,7 @@ func TestDynamicClustersNotReadyNamespace(t *testing.T) { }}) requireClusterNamespaceIDs(t, clusters, []ident.ID{defaultTestNs1ID}) + requireNonReadyClusterNamespaceIDs(t, clusters, []ident.ID{defaultTestNs2ID, ident.StringID("foo")}) } func TestDynamicClustersEmptyNamespacesThenUpdates(t *testing.T) { @@ -373,7 +374,7 @@ func TestDynamicClustersEmptyNamespacesThenUpdates(t *testing.T) { Retention: 48 * time.Hour, }}) - return found && assertClusterNamespaceIDs(clusters, []ident.ID{defaultTestNs1ID, defaultTestNs2ID}) + return found && assertClusterNamespaceIDs(clusters.ClusterNamespaces(), []ident.ID{defaultTestNs1ID, defaultTestNs2ID}) }, time.Second)) } @@ -434,16 +435,20 @@ type mapParams struct { nsOpts namespace.Options } +func requireNonReadyClusterNamespaceIDs(t *testing.T, clusters Clusters, ids []ident.ID) { + require.True(t, assertClusterNamespaceIDs(clusters.NonReadyClusterNamespaces(), ids)) +} + func requireClusterNamespaceIDs(t *testing.T, clusters Clusters, ids []ident.ID) { - require.True(t, assertClusterNamespaceIDs(clusters, ids)) + require.True(t, assertClusterNamespaceIDs(clusters.ClusterNamespaces(), ids)) } -func assertClusterNamespaceIDs(clusters Clusters, ids []ident.ID) bool { +func assertClusterNamespaceIDs(actual ClusterNamespaces, ids []ident.ID) bool { var ( nsIds = make([]string, 0, len(ids)) expectedIds = make([]string, 0, len(ids)) ) - for _, ns := range clusters.ClusterNamespaces() { + for _, ns := range actual { nsIds = append(nsIds, ns.NamespaceID().String()) } for _, id := range ids { From 99f874ff5949abf240ada29e3b7bfdb3ea345aa9 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Tue, 3 Nov 2020 11:57:44 -0500 Subject: [PATCH 20/47] [tools] Add concurrent read_index_segments validation option (#2833) --- .../tools/read_index_segments/main/main.go | 354 +++++++++++------- 1 file changed, 209 insertions(+), 145 deletions(-) diff --git a/src/cmd/tools/read_index_segments/main/main.go b/src/cmd/tools/read_index_segments/main/main.go index 680e8019cc..7a927d9c9d 100644 --- a/src/cmd/tools/read_index_segments/main/main.go +++ b/src/cmd/tools/read_index_segments/main/main.go @@ -21,28 +21,38 @@ package main import ( - "io" - - "github.com/m3db/m3/src/x/unsafe" - "fmt" + "io" + "io/ioutil" golog "log" + "math" "os" + "runtime" + "sync" "time" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/query/util/json" "github.com/m3db/m3/src/x/ident" + xsync "github.com/m3db/m3/src/x/sync" + "github.com/m3db/m3/src/x/unsafe" "github.com/pborman/getopt" "go.uber.org/zap" ) +var ( + halfCPUs = int(math.Max(float64(runtime.NumCPU()/2), 1)) + endlineBytes = []byte("\n") +) + func main() { var ( - optPathPrefix = getopt.StringLong("path-prefix", 'p', "/var/lib/m3db", "Path prefix [e.g. /var/lib/m3db]") - optOutputFile = getopt.StringLong("output-file", 'o', "", "Output JSON file of line delimited JSON objects for each segment") + optPathPrefix = getopt.StringLong("path-prefix", 'p', "/var/lib/m3db", "Path prefix [e.g. /var/lib/m3db]") + optOutputFile = getopt.StringLong("output-file", 'o', "", "Output JSON file of line delimited JSON objects for each segment") + optValidate = getopt.BoolLong("validate", 'v', "Validate the segments, do not print out metadata") + optValidateConcurrency = getopt.IntLong("validate-concurrency", 'c', halfCPUs, "Validation concurrency") ) getopt.Parse() @@ -52,23 +62,33 @@ func main() { golog.Fatalf("unable to create logger: %+v", err) } - if *optPathPrefix == "" || *optOutputFile == "" { + if *optOutputFile != "" && *optValidate { + log.Error("cannot write output and validate, do not set output file if validating") + getopt.Usage() + os.Exit(1) + } + + if *optPathPrefix == "" || (*optOutputFile == "" && !*optValidate) { getopt.Usage() os.Exit(1) } run(runOptions{ - filePathPrefix: *optPathPrefix, - outputFilePath: *optOutputFile, - log: log, + filePathPrefix: *optPathPrefix, + outputFilePath: *optOutputFile, + validate: *optValidate, + validateConcurrency: *optValidateConcurrency, + log: log, }) } type runOptions struct { - filePathPrefix string - outputFilePath string - log *zap.Logger + filePathPrefix string + outputFilePath string + validate bool + validateConcurrency int + log *zap.Logger } func run(opts runOptions) { @@ -87,21 +107,38 @@ func run(opts runOptions) { // Get all fileset files. log.Info("discovered namespaces", zap.Strings("namespaces", namespaces)) - out, err := os.Create(opts.outputFilePath) - if err != nil { - log.Fatal("unable to create output file", - zap.String("file", opts.outputFilePath), - zap.Error(err)) + var ( + out io.Writer + validateWorkerPool xsync.WorkerPool + ) + if opts.validate { + // Only validating, output to dev null. + out = ioutil.Discard + validateWorkerPool = xsync.NewWorkerPool(opts.validateConcurrency) + validateWorkerPool.Init() + log.Info("validating segment files", + zap.Int("concurrency", opts.validateConcurrency)) + } else { + // Output to file. + out, err = os.Create(opts.outputFilePath) + if err != nil { + log.Fatal("unable to create output file", + zap.String("file", opts.outputFilePath), + zap.Error(err)) + } + log.Info("writing output JSON line delimited", + zap.String("path", opts.outputFilePath)) } for _, namespace := range namespaces { log.Info("reading segments", zap.String("namespace", namespace)) ns := ident.StringID(namespace) - readNamespaceSegments(out, ns, fsOpts, log) + readNamespaceSegments(out, opts.validate, validateWorkerPool, + ns, fsOpts, log) // Separate by endline. - if _, err := out.WriteString("\n"); err != nil { + if _, err := out.Write(endlineBytes); err != nil { log.Fatal("could not write endline", zap.Error(err)) } } @@ -109,12 +146,17 @@ func run(opts runOptions) { func readNamespaceSegments( out io.Writer, + validate bool, + validateWorkerPool xsync.WorkerPool, nsID ident.ID, fsOpts fs.Options, log *zap.Logger, ) { - infoFiles := fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID, - fsOpts.InfoReaderBufferSize()) + var ( + infoFiles = fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID, + fsOpts.InfoReaderBufferSize()) + wg sync.WaitGroup + ) for _, infoFile := range infoFiles { if err := infoFile.Err.Error(); err != nil { @@ -126,180 +168,202 @@ func readNamespaceSegments( continue } - log.Info("reading block segments", - zap.String("namespace", nsID.String()), - zap.String("blockStart", infoFile.ID.BlockStart.String()), - zap.Int64("blockStartUnixNano", infoFile.ID.BlockStart.UnixNano()), - zap.Strings("files", infoFile.AbsoluteFilePaths)) - - segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ - ReaderOptions: fs.IndexReaderOpenOptions{ - Identifier: infoFile.ID, - FileSetType: persist.FileSetFlushType, - }, - FilesystemOptions: fsOpts, - }) - if err != nil { - log.Error("unable to read segments from index fileset", - zap.Stringer("namespace", nsID), - zap.Error(err), - zap.Time("blockStart", time.Unix(0, infoFile.Info.BlockStart)), - zap.Int("volumeIndex", infoFile.ID.VolumeIndex), - ) + if !validate { + readBlockSegments(out, nsID, infoFile, fsOpts, log) continue } - for i, seg := range segments { - jw := json.NewWriter(out) - jw.BeginObject() + // Validating, so use validation concurrency. + wg.Add(1) + validateWorkerPool.Go(func() { + defer wg.Done() + readBlockSegments(out, nsID, infoFile, fsOpts, log) + }) + } - jw.BeginObjectField("namespace") - jw.WriteString(nsID.String()) + // Wait for any concurrent validation. + wg.Wait() +} - jw.BeginObjectField("blockStart") - jw.WriteString(time.Unix(0, infoFile.Info.BlockStart).Format(time.RFC3339)) +func readBlockSegments( + out io.Writer, + nsID ident.ID, + infoFile fs.ReadIndexInfoFileResult, + fsOpts fs.Options, + log *zap.Logger, +) { + // Make sure if we fatal or error out the exact block is known. + log = log.With( + zap.String("namespace", nsID.String()), + zap.String("blockStart", infoFile.ID.BlockStart.String()), + zap.Int64("blockStartUnixNano", infoFile.ID.BlockStart.UnixNano()), + zap.Int("volumeIndex", infoFile.ID.VolumeIndex), + zap.Strings("files", infoFile.AbsoluteFilePaths)) + + log.Info("reading block segments") + + segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ + ReaderOptions: fs.IndexReaderOpenOptions{ + Identifier: infoFile.ID, + FileSetType: persist.FileSetFlushType, + }, + FilesystemOptions: fsOpts, + }) + if err != nil { + log.Error("unable to read segments from index fileset", zap.Error(err)) + return + } - jw.BeginObjectField("volumeIndex") - jw.WriteInt(infoFile.ID.VolumeIndex) + for i, seg := range segments { + jw := json.NewWriter(out) + jw.BeginObject() - jw.BeginObjectField("segmentIndex") - jw.WriteInt(i) + jw.BeginObjectField("namespace") + jw.WriteString(nsID.String()) - reader, err := seg.Reader() - if err != nil { - log.Fatal("unable to create segment reader", zap.Error(err)) - } + jw.BeginObjectField("blockStart") + jw.WriteString(time.Unix(0, infoFile.Info.BlockStart).Format(time.RFC3339)) - iter, err := reader.AllDocs() - if err != nil { - log.Fatal("unable to iterate segment docs", zap.Error(err)) - } + jw.BeginObjectField("volumeIndex") + jw.WriteInt(infoFile.ID.VolumeIndex) - jw.BeginObjectField("documents") - jw.BeginArray() - for postingsID := 0; iter.Next(); postingsID++ { - d := iter.Current() - jw.BeginObject() + jw.BeginObjectField("segmentIndex") + jw.WriteInt(i) + + reader, err := seg.Reader() + if err != nil { + log.Fatal("unable to create segment reader", zap.Error(err)) + } - jw.BeginObjectField("postingsID") - jw.WriteInt(postingsID) + iter, err := reader.AllDocs() + if err != nil { + log.Fatal("unable to iterate segment docs", zap.Error(err)) + } - jw.BeginObjectField("id") - unsafe.WithString(d.ID, func(str string) { - jw.WriteString(str) - }) + jw.BeginObjectField("documents") + jw.BeginArray() + for postingsID := 0; iter.Next(); postingsID++ { + d := iter.Current() + jw.BeginObject() - jw.BeginObjectField("fields") + jw.BeginObjectField("postingsID") + jw.WriteInt(postingsID) - jw.BeginArray() - for _, field := range d.Fields { - jw.BeginObject() + jw.BeginObjectField("id") + unsafe.WithString(d.ID, func(str string) { + jw.WriteString(str) + }) - jw.BeginObjectField("name") - unsafe.WithString(field.Name, func(str string) { - jw.WriteString(str) - }) + jw.BeginObjectField("fields") + + jw.BeginArray() + for _, field := range d.Fields { + jw.BeginObject() - jw.BeginObjectField("value") - unsafe.WithString(field.Name, func(str string) { - jw.WriteString(str) - }) + jw.BeginObjectField("name") + unsafe.WithString(field.Name, func(str string) { + jw.WriteString(str) + }) - jw.EndObject() - } - jw.EndArray() + jw.BeginObjectField("value") + unsafe.WithString(field.Name, func(str string) { + jw.WriteString(str) + }) jw.EndObject() } jw.EndArray() - if err := iter.Err(); err != nil { - log.Fatal("doc iterator error", zap.Error(err)) - } - if err := iter.Close(); err != nil { - log.Fatal("doc iterator close error", zap.Error(err)) - } + jw.EndObject() + } + jw.EndArray() - fieldsIter, err := seg.FieldsIterable().Fields() + if err := iter.Err(); err != nil { + log.Fatal("doc iterator error", zap.Error(err)) + } + if err := iter.Close(); err != nil { + log.Fatal("doc iterator close error", zap.Error(err)) + } + + fieldsIter, err := seg.FieldsIterable().Fields() + if err != nil { + log.Fatal("could not create fields iterator", zap.Error(err)) + } + + jw.BeginObjectField("fields") + jw.BeginArray() + for fieldsIter.Next() { + field := fieldsIter.Current() + + jw.BeginObject() + jw.BeginObjectField("field") + unsafe.WithString(field, func(str string) { + jw.WriteString(str) + }) + + termsIter, err := seg.TermsIterable().Terms(field) if err != nil { - log.Fatal("could not create fields iterator", zap.Error(err)) + log.Fatal("could not create terms iterator", zap.Error(err)) } - jw.BeginObjectField("fields") + jw.BeginObjectField("terms") jw.BeginArray() - for fieldsIter.Next() { - field := fieldsIter.Current() + for termsIter.Next() { + term, postingsList := termsIter.Current() jw.BeginObject() - jw.BeginObjectField("field") - unsafe.WithString(field, func(str string) { + jw.BeginObjectField("term") + unsafe.WithString(term, func(str string) { jw.WriteString(str) }) - termsIter, err := seg.TermsIterable().Terms(field) - if err != nil { - log.Fatal("could not create terms iterator", zap.Error(err)) - } + postingsIter := postingsList.Iterator() - jw.BeginObjectField("terms") + jw.BeginObjectField("postings") jw.BeginArray() - for termsIter.Next() { - term, postingsList := termsIter.Current() - - jw.BeginObject() - jw.BeginObjectField("term") - unsafe.WithString(term, func(str string) { - jw.WriteString(str) - }) - - postingsIter := postingsList.Iterator() - - jw.BeginObjectField("postings") - jw.BeginArray() - for postingsIter.Next() { - postingsID := postingsIter.Current() - jw.WriteInt(int(postingsID)) - } - jw.EndArray() - jw.EndObject() - - if err := postingsIter.Err(); err != nil { - log.Fatal("postings iterator error", zap.Error(err)) - } - - if err := postingsIter.Close(); err != nil { - log.Fatal("postings iterator close error", zap.Error(err)) - } + for postingsIter.Next() { + postingsID := postingsIter.Current() + jw.WriteInt(int(postingsID)) } jw.EndArray() jw.EndObject() - if err := termsIter.Err(); err != nil { - log.Fatal("field iterator error", zap.Error(err)) + if err := postingsIter.Err(); err != nil { + log.Fatal("postings iterator error", zap.Error(err)) } - if err := termsIter.Close(); err != nil { - log.Fatal("field iterator close error", zap.Error(err)) + if err := postingsIter.Close(); err != nil { + log.Fatal("postings iterator close error", zap.Error(err)) } } jw.EndArray() + jw.EndObject() - if err := fieldsIter.Err(); err != nil { + if err := termsIter.Err(); err != nil { log.Fatal("field iterator error", zap.Error(err)) } - if err := fieldsIter.Close(); err != nil { + if err := termsIter.Close(); err != nil { log.Fatal("field iterator close error", zap.Error(err)) } + } + jw.EndArray() - jw.EndObject() + if err := fieldsIter.Err(); err != nil { + log.Fatal("field iterator error", zap.Error(err)) + } - if err := jw.Flush(); err != nil { - log.Fatal("could not flush JSON writer", zap.Error(err)) - } - if err := jw.Close(); err != nil { - log.Fatal("could not close JSON writer", zap.Error(err)) - } + if err := fieldsIter.Close(); err != nil { + log.Fatal("field iterator close error", zap.Error(err)) + } + + jw.EndObject() + + if err := jw.Flush(); err != nil { + log.Fatal("could not flush JSON writer", zap.Error(err)) + } + if err := jw.Close(); err != nil { + log.Fatal("could not close JSON writer", zap.Error(err)) } } } From 2ac4853aa50b8618a9fcf1788b7048c731614bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Tue, 3 Nov 2020 20:17:51 +0200 Subject: [PATCH 21/47] Add a check for seriesIter.Err (#2834) --- src/dbnode/storage/shard.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dbnode/storage/shard.go b/src/dbnode/storage/shard.go index 46f3d892ea..292a5dfb05 100644 --- a/src/dbnode/storage/shard.go +++ b/src/dbnode/storage/shard.go @@ -2937,6 +2937,10 @@ func encodeAggregatedSeries( processedTileCount++ } + if err := seriesIter.Err(); err != nil { + return 0, err + } + return processedTileCount, nil } From 5fdf1c216a46dd2e5d8cafcd675cc2f8afdc0564 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Thu, 5 Nov 2020 06:34:04 +1100 Subject: [PATCH 22/47] [query] Return additional warnings in /query{,_range} endpoints (#2836) --- src/query/api/v1/handler/prom/read.go | 5 +++++ src/query/api/v1/handler/prom/read_instant.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/query/api/v1/handler/prom/read.go b/src/query/api/v1/handler/prom/read.go index dc8c49e718..24f69c2986 100644 --- a/src/query/api/v1/handler/prom/read.go +++ b/src/query/api/v1/handler/prom/read.go @@ -24,6 +24,7 @@ package prom import ( "context" + "errors" "net/http" "time" @@ -120,6 +121,10 @@ func (h *readHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + for _, warn := range resultMetadata.Warnings { + res.Warnings = append(res.Warnings, errors.New(warn.Message)) + } + query := request.Params.Query err = ApplyRangeWarnings(query, &resultMetadata) if err != nil { diff --git a/src/query/api/v1/handler/prom/read_instant.go b/src/query/api/v1/handler/prom/read_instant.go index ad77836bee..17faf533a7 100644 --- a/src/query/api/v1/handler/prom/read_instant.go +++ b/src/query/api/v1/handler/prom/read_instant.go @@ -22,6 +22,7 @@ package prom import ( "context" + "errors" "fmt" "net/http" "time" @@ -119,6 +120,10 @@ func (h *readInstantHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + for _, warn := range resultMetadata.Warnings { + res.Warnings = append(res.Warnings, errors.New(warn.Message)) + } + err = ApplyRangeWarnings(query, &resultMetadata) if err != nil { h.logger.Warn("error applying range warnings", From 158ab107b3e5e08948462f59a842327cede1acde Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 4 Nov 2020 18:03:10 -0500 Subject: [PATCH 23/47] [read_index_segments] Always validate index segment checksums before reading/validating contents (#2835) --- Makefile | 1 + .../tools/query_index_segments/main/main.go | 256 ++++++++++++++++++ .../tools/read_index_segments/main/main.go | 16 +- src/cmd/tools/verify_data_files/main/main.go | 14 +- src/dbnode/persist/fs/index_read_segments.go | 30 +- src/dbnode/persist/fs/persist_manager.go | 7 +- .../bootstrap/bootstrapper/fs/source.go | 6 +- 7 files changed, 314 insertions(+), 16 deletions(-) create mode 100644 src/cmd/tools/query_index_segments/main/main.go diff --git a/Makefile b/Makefile index e208a0ab93..c737c4d4f8 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,7 @@ TOOLS := \ read_data_files \ read_index_files \ read_index_segments \ + query_index_segments \ clone_fileset \ dtest \ verify_data_files \ diff --git a/src/cmd/tools/query_index_segments/main/main.go b/src/cmd/tools/query_index_segments/main/main.go new file mode 100644 index 0000000000..1556a4add4 --- /dev/null +++ b/src/cmd/tools/query_index_segments/main/main.go @@ -0,0 +1,256 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "errors" + golog "log" + "math" + "os" + "runtime" + "sync" + "time" + + "github.com/m3db/m3/src/dbnode/persist" + "github.com/m3db/m3/src/dbnode/persist/fs" + "github.com/m3db/m3/src/m3ninx/index" + "github.com/m3db/m3/src/m3ninx/search/executor" + "github.com/m3db/m3/src/query/generated/proto/prompb" + "github.com/m3db/m3/src/query/parser/promql" + "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/x/ident" + xsync "github.com/m3db/m3/src/x/sync" + + "github.com/pborman/getopt" + "github.com/prometheus/prometheus/pkg/labels" + "go.uber.org/zap" +) + +var ( + halfCPUs = int(math.Max(float64(runtime.NumCPU()/2), 1)) + endlineBytes = []byte("\n") +) + +func main() { + + var ( + optPathPrefix = getopt.StringLong("path-prefix", 'p', "/var/lib/m3db", "Path prefix [e.g. /var/lib/m3db]") + optNamespace = getopt.StringLong("namespace", 'n', "", "Namespace to query") + optQuery = getopt.StringLong("query", 'q', "", "Query to issue to match time series (PromQL selector)") + optConcurrency = getopt.IntLong("concurrency", 'c', halfCPUs, "Query concurrency") + optValidate = true + ) + getopt.BoolVarLong(&optValidate, "validate", 'v', "Validate index segments before querying") + getopt.Parse() + + logConfig := zap.NewDevelopmentConfig() + log, err := logConfig.Build() + if err != nil { + golog.Fatalf("unable to create logger: %+v", err) + } + + if *optNamespace == "" || *optQuery == "" { + getopt.Usage() + os.Exit(1) + } + + run(runOptions{ + filePathPrefix: *optPathPrefix, + namespace: *optNamespace, + query: *optQuery, + validate: optValidate, + concurrency: *optConcurrency, + log: log, + }) + +} + +type runOptions struct { + filePathPrefix string + namespace string + query string + validate bool + concurrency int + log *zap.Logger +} + +func run(opts runOptions) { + log := opts.log + + parseOpts := promql.NewParseOptions() + parse := parseOpts.MetricSelectorFn() + + matchers, err := parse(opts.query) + if err != nil { + log.Fatal("could not create matchers", zap.Error(err)) + } + + labelMatchers, err := toLabelMatchers(matchers) + if err != nil { + log.Fatal("could not create label matchers", zap.Error(err)) + } + + query, err := storage.PromReadQueryToM3(&prompb.Query{ + Matchers: labelMatchers, + StartTimestampMs: 0, + EndTimestampMs: time.Now().UnixNano() / int64(time.Millisecond), + }) + if err != nil { + log.Fatal("could not create M3 fetch query", zap.Error(err)) + } + + indexQuery, err := storage.FetchQueryToM3Query(query, storage.NewFetchOptions()) + if err != nil { + log.Fatal("could not create M3 index query", zap.Error(err)) + } + + fsOpts := fs.NewOptions(). + SetFilePathPrefix(opts.filePathPrefix) + + if opts.validate { + // Validate checksums before reading and/or validating contents if set. + fsOpts = fsOpts.SetIndexReaderAutovalidateIndexSegments(true) + } + + var ( + nsID = ident.StringID(opts.namespace) + infoFiles = fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID, + fsOpts.InfoReaderBufferSize()) + results = make(map[string]struct{}) + resultsLock sync.Mutex + wg sync.WaitGroup + ) + + log.Info("starting query", + zap.Int("concurrency", opts.concurrency), + zap.Bool("validateSegments", opts.validate)) + + workers := xsync.NewWorkerPool(opts.concurrency) + workers.Init() + + for _, infoFile := range infoFiles { + if err := infoFile.Err.Error(); err != nil { + log.Error("unable to read index info file", + zap.Stringer("namespace", nsID), + zap.Error(err), + zap.String("filepath", infoFile.Err.Filepath()), + ) + continue + } + + readResult, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ + ReaderOptions: fs.IndexReaderOpenOptions{ + Identifier: infoFile.ID, + FileSetType: persist.FileSetFlushType, + }, + FilesystemOptions: fsOpts, + }) + if err != nil { + log.Fatal("unable to read segments from index fileset", zap.Error(err)) + return + } + + wg.Add(1) + workers.Go(func() { + defer wg.Done() + + var readers []index.Reader + for _, seg := range readResult.Segments { + reader, err := seg.Reader() + if err != nil { + log.Fatal("segment reader error", zap.Error(err)) + } + + readers = append(readers, reader) + } + + executor := executor.NewExecutor(readers) + + iter, err := executor.Execute(indexQuery.Query.SearchQuery()) + if err != nil { + log.Fatal("search execute error", zap.Error(err)) + } + + fields := make(map[string]string) + for iter.Next() { + d := iter.Current() + + key := string(d.ID) + + resultsLock.Lock() + _, ok := results[key] + if !ok { + results[key] = struct{}{} + } + resultsLock.Unlock() + + if ok { + continue // Already printed. + } + + for k := range fields { + delete(fields, k) + } + for _, field := range d.Fields { + fields[string(field.Name)] = string(field.Value) + } + + log.Info("matched document", + zap.String("id", key), + zap.Any("fields", fields)) + } + + if err := iter.Err(); err != nil { + log.Fatal("iterate err", zap.Error(err)) + } + if err := iter.Close(); err != nil { + log.Fatal("iterate close err", zap.Error(err)) + } + }) + } + + wg.Wait() +} + +func toLabelMatchers(matchers []*labels.Matcher) ([]*prompb.LabelMatcher, error) { + pbMatchers := make([]*prompb.LabelMatcher, 0, len(matchers)) + for _, m := range matchers { + var mType prompb.LabelMatcher_Type + switch m.Type { + case labels.MatchEqual: + mType = prompb.LabelMatcher_EQ + case labels.MatchNotEqual: + mType = prompb.LabelMatcher_NEQ + case labels.MatchRegexp: + mType = prompb.LabelMatcher_RE + case labels.MatchNotRegexp: + mType = prompb.LabelMatcher_NRE + default: + return nil, errors.New("invalid matcher type") + } + pbMatchers = append(pbMatchers, &prompb.LabelMatcher{ + Type: mType, + Name: []byte(m.Name), + Value: []byte(m.Value), + }) + } + return pbMatchers, nil +} diff --git a/src/cmd/tools/read_index_segments/main/main.go b/src/cmd/tools/read_index_segments/main/main.go index 7a927d9c9d..ef904b795c 100644 --- a/src/cmd/tools/read_index_segments/main/main.go +++ b/src/cmd/tools/read_index_segments/main/main.go @@ -95,7 +95,11 @@ func run(opts runOptions) { log := opts.log fsOpts := fs.NewOptions(). - SetFilePathPrefix(opts.filePathPrefix) + SetFilePathPrefix(opts.filePathPrefix). + // Always validate checksums before reading and/or validating contents + // regardless of whether this is a validation run or just reading + // the raw files. + SetIndexReaderAutovalidateIndexSegments(true) indexDirPath := fs.IndexDataDirPath(opts.filePathPrefix) @@ -202,7 +206,7 @@ func readBlockSegments( log.Info("reading block segments") - segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ + readResult, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ ReaderOptions: fs.IndexReaderOpenOptions{ Identifier: infoFile.ID, FileSetType: persist.FileSetFlushType, @@ -214,7 +218,13 @@ func readBlockSegments( return } - for i, seg := range segments { + if readResult.Validated { + log.Info("validated segments") + } else { + log.Error("expected to validate segments but did not validate") + } + + for i, seg := range readResult.Segments { jw := json.NewWriter(out) jw.BeginObject() diff --git a/src/cmd/tools/verify_data_files/main/main.go b/src/cmd/tools/verify_data_files/main/main.go index 9ac107f650..8cae70fa51 100644 --- a/src/cmd/tools/verify_data_files/main/main.go +++ b/src/cmd/tools/verify_data_files/main/main.go @@ -165,6 +165,10 @@ func run(opts runOptions) { log.Info("verifying file sets", zap.Int("numFileSets", len(fileSetFiles))) for _, fileSet := range fileSetFiles { + if !fileSet.HasCompleteCheckpointFile() { + continue // Don't validate file sets without checkpoint file. + } + log.Info("verifying file set file", zap.Any("fileSet", fileSet)) if err := verifyFileSet(verifyFileSetOptions{ filePathPrefix: filePathPrefix, @@ -301,10 +305,16 @@ func readEntry( data checked.Bytes, checksum uint32, ) (readEntryResult, error) { - if !utf8.Valid(id.Bytes()) { + idValue := id.Bytes() + if len(idValue) == 0 { + return readEntryResult{invalidID: true}, + fmt.Errorf("invalid id: err=%s, as_string=%s, as_hex=%x", + "empty", idValue, idValue) + } + if !utf8.Valid(idValue) { return readEntryResult{invalidID: true}, fmt.Errorf("invalid id: err=%s, as_string=%s, as_hex=%x", - "non-utf8", id.Bytes(), id.Bytes()) + "non-utf8", idValue, idValue) } for tags.Next() { diff --git a/src/dbnode/persist/fs/index_read_segments.go b/src/dbnode/persist/fs/index_read_segments.go index b31070b499..73a398b792 100644 --- a/src/dbnode/persist/fs/index_read_segments.go +++ b/src/dbnode/persist/fs/index_read_segments.go @@ -47,14 +47,20 @@ type ReadIndexSegmentsOptions struct { newPersistentSegmentFn newPersistentSegmentFn } +// ReadIndexSegmentsResult is the result of a call to ReadIndexSegments. +type ReadIndexSegmentsResult struct { + Segments []segment.Segment + Validated bool +} + // ReadIndexSegments will read a set of segments. func ReadIndexSegments( opts ReadIndexSegmentsOptions, -) ([]segment.Segment, error) { +) (ReadIndexSegmentsResult, error) { readerOpts := opts.ReaderOptions fsOpts := opts.FilesystemOptions if fsOpts == nil { - return nil, errFilesystemOptionsNotSpecified + return ReadIndexSegmentsResult{}, errFilesystemOptionsNotSpecified } newReader := opts.newReaderFn @@ -69,14 +75,21 @@ func ReadIndexSegments( reader, err := newReader(fsOpts) if err != nil { - return nil, err + return ReadIndexSegmentsResult{}, err } var ( segments []segment.Segment + validate = opts.FilesystemOptions.IndexReaderAutovalidateIndexSegments() success = false ) + if validate { + if err = reader.Validate(); err != nil { + return ReadIndexSegmentsResult{}, err + } + } + // Need to do this to guarantee we release all resources in case of failure. defer func() { if !success { @@ -88,7 +101,7 @@ func ReadIndexSegments( }() if _, err := reader.Open(readerOpts); err != nil { - return nil, err + return ReadIndexSegmentsResult{}, err } segments = make([]segment.Segment, 0, reader.SegmentFileSets()) @@ -98,13 +111,13 @@ func ReadIndexSegments( break } if err != nil { - return nil, err + return ReadIndexSegmentsResult{}, err } fstOpts := fsOpts.FSTOptions() seg, err := newPersistentSegment(fileset, fstOpts) if err != nil { - return nil, err + return ReadIndexSegmentsResult{}, err } segments = append(segments, seg) @@ -113,5 +126,8 @@ func ReadIndexSegments( // Indicate we don't need the defer() above to release any resources, as we are // transferring ownership to the caller. success = true - return segments, nil + return ReadIndexSegmentsResult{ + Segments: segments, + Validated: validate, + }, nil } diff --git a/src/dbnode/persist/fs/persist_manager.go b/src/dbnode/persist/fs/persist_manager.go index 752e48f656..69a45d2bb2 100644 --- a/src/dbnode/persist/fs/persist_manager.go +++ b/src/dbnode/persist/fs/persist_manager.go @@ -342,7 +342,7 @@ func (pm *persistManager) closeIndex() ([]segment.Segment, error) { // and then we get persistent segments backed by mmap'd data so the index // can safely evict the segment's we have just persisted. - return ReadIndexSegments(ReadIndexSegmentsOptions{ + result, err := ReadIndexSegments(ReadIndexSegmentsOptions{ ReaderOptions: IndexReaderOpenOptions{ Identifier: pm.indexPM.fileSetIdentifier, FileSetType: pm.indexPM.fileSetType, @@ -351,6 +351,11 @@ func (pm *persistManager) closeIndex() ([]segment.Segment, error) { newReaderFn: pm.indexPM.newReaderFn, newPersistentSegmentFn: pm.indexPM.newPersistentSegmentFn, }) + if err != nil { + return nil, err + } + + return result.Segments, nil } // DoneIndex is called by the databaseFlushManager to finish the index persist process. diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go index 07943e89d2..e72c05c1cd 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go @@ -982,7 +982,7 @@ func (s *fileSystemSource) bootstrapFromIndexPersistedBlocks( continue } - segments, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ + readResult, err := fs.ReadIndexSegments(fs.ReadIndexSegmentsOptions{ ReaderOptions: fs.IndexReaderOpenOptions{ Identifier: infoFile.ID, FileSetType: persist.FileSetFlushType, @@ -1008,8 +1008,8 @@ func (s *fileSystemSource) bootstrapFromIndexPersistedBlocks( } segmentsFulfilled := willFulfill // NB(bodu): All segments read from disk are already persisted. - persistedSegments := make([]result.Segment, 0, len(segments)) - for _, segment := range segments { + persistedSegments := make([]result.Segment, 0, len(readResult.Segments)) + for _, segment := range readResult.Segments { persistedSegments = append(persistedSegments, result.NewSegment(segment, true)) } volumeType := idxpersist.DefaultIndexVolumeType From 4129dbb1ed246c0e6fa4b1b277c35d2d32820db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=BBinas?= Date: Thu, 5 Nov 2020 10:18:18 +0200 Subject: [PATCH 24/47] [query] sort, topk, bottomk fixes for instant queries (#2792) * fixes most of topk, bottomk ordered tests when instant queries are used * formatted some code and extracted instant logic into separate method * Changes after review. Small optimization in appendValuesInstant. * fixed imports. made valueAndMeta struct package private. * Optimized instant queries - now only single block will be created. OrderedFlush() now sorts the results instead popping them one by one. FloatHeap won't be created if it's not used. * improved how max series count is calculated * removed unused method * initial implementation * Fixes NaN issues with m3 queries for sort and topk/bottomk. FloatHeap now supports adding NaN values. Uncommented topk, bottomk and sort compatibility tests (now they pass). * Retain old KeepNans flag for non instant queries * Changes after code review. * Small cleanup. * Added copyright header to sort_test.go. equalsPairs now checks individual struct elements instead of their printed values. * better init * Removed Min() function in favour of using this logic inline. Used append in place when indices are not needed. Documented and extracted some comparison functions for better readability and clearness. * Updated doc comment --- src/cmd/services/m3query/config/config.go | 4 +- .../services/m3query/config/config_test.go | 6 +- .../api/v1/handler/prometheus/native/read.go | 9 +- .../handler/prometheus/native/read_common.go | 1 + .../v1/handler/prometheus/native/read_test.go | 4 +- .../v1/handler/prometheus/remote/read_test.go | 4 +- src/query/block/meta.go | 2 + src/query/functions/aggregation/take.go | 184 ++++++++++++++---- src/query/functions/aggregation/take_test.go | 72 +++++++ src/query/functions/linear/sort.go | 141 +++++++++++++- src/query/functions/linear/sort_test.go | 141 ++++++++++++++ src/query/functions/utils/heap.go | 54 +++-- src/query/functions/utils/heap_test.go | 135 +++++++++++++ src/query/models/query_context.go | 3 + src/query/parser/promql/matchers.go | 5 +- src/query/parser/promql/parse_test.go | 6 +- .../compatibility/testdata/aggregators.test | 85 ++++---- .../compatibility/testdata/functions.test | 43 ++-- 18 files changed, 766 insertions(+), 133 deletions(-) create mode 100644 src/query/functions/linear/sort_test.go diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 3d1a0adfcf..e1da5d5e13 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -203,9 +203,9 @@ type FilterConfiguration struct { // ResultOptions are the result options for query. type ResultOptions struct { - // KeepNans keeps NaNs before returning query results. + // KeepNaNs keeps NaNs before returning query results. // The default is false, which matches Prometheus - KeepNans bool `yaml:"keepNans"` + KeepNaNs bool `yaml:"keepNans"` } // QueryConfiguration is the query configuration. diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index acb787da8c..88bb05855d 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -185,10 +185,10 @@ func TestTagOptionsConfig(t *testing.T) { func TestKeepNaNsDefault(t *testing.T) { r := ResultOptions{ - KeepNans: true, + KeepNaNs: true, } - assert.Equal(t, true, r.KeepNans) + assert.Equal(t, true, r.KeepNaNs) r = ResultOptions{} - assert.Equal(t, false, r.KeepNans) + assert.Equal(t, false, r.KeepNaNs) } diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go index 9db53457b1..cbb72f4ad3 100644 --- a/src/query/api/v1/handler/prometheus/native/read.go +++ b/src/query/api/v1/handler/prometheus/native/read.go @@ -141,15 +141,20 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { handleroptions.AddWarningHeaders(w, result.Meta) h.promReadMetrics.fetchSuccess.Inc(1) + keepNaNs := h.opts.Config().ResultOptions.KeepNaNs + if !keepNaNs { + keepNaNs = result.Meta.KeepNaNs + } + if h.instant { - renderResultsInstantaneousJSON(w, result, h.opts.Config().ResultOptions.KeepNans) + renderResultsInstantaneousJSON(w, result, keepNaNs) return } err = RenderResultsJSON(w, result, RenderResultsOptions{ Start: parsedOptions.Params.Start, End: parsedOptions.Params.End, - KeepNaNs: h.opts.Config().ResultOptions.KeepNans, + KeepNaNs: h.opts.Config().ResultOptions.KeepNaNs, }) if err != nil { diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index 44cb14662a..f86ba1a304 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -101,6 +101,7 @@ func parseRequest( QueryContextOptions: models.QueryContextOptions{ LimitMaxTimeseries: fetchOpts.SeriesLimit, LimitMaxDocs: fetchOpts.DocsLimit, + Instantaneous: instantaneous, }} restrictOpts := fetchOpts.RestrictQueryOptions.GetRestrictByType() diff --git a/src/query/api/v1/handler/prometheus/native/read_test.go b/src/query/api/v1/handler/prometheus/native/read_test.go index 5ae88efae2..c1b3bffca9 100644 --- a/src/query/api/v1/handler/prometheus/native/read_test.go +++ b/src/query/api/v1/handler/prometheus/native/read_test.go @@ -208,7 +208,7 @@ func newTestSetup( fetchOptsBuilder := handleroptions.NewFetchOptionsBuilder(fetchOptsBuilderCfg) tagOpts := models.NewTagOptions() limitsConfig := config.LimitsConfiguration{} - keepNans := false + keepNaNs := false opts := options.EmptyHandlerOptions(). SetEngine(engine). @@ -219,7 +219,7 @@ func newTestSetup( SetConfig(config.Configuration{ Limits: limitsConfig, ResultOptions: config.ResultOptions{ - KeepNans: keepNans, + KeepNaNs: keepNaNs, }, }) diff --git a/src/query/api/v1/handler/prometheus/remote/read_test.go b/src/query/api/v1/handler/prometheus/remote/read_test.go index c8c2504211..9ded0bdc53 100644 --- a/src/query/api/v1/handler/prometheus/remote/read_test.go +++ b/src/query/api/v1/handler/prometheus/remote/read_test.go @@ -376,7 +376,7 @@ func TestMultipleRead(t *testing.T) { handlerOpts := options.EmptyHandlerOptions().SetEngine(engine). SetConfig(config.Configuration{ ResultOptions: config.ResultOptions{ - KeepNans: true, + KeepNaNs: true, }, }) @@ -454,7 +454,7 @@ func TestReadWithOptions(t *testing.T) { handlerOpts := options.EmptyHandlerOptions().SetEngine(engine). SetConfig(config.Configuration{ ResultOptions: config.ResultOptions{ - KeepNans: true, + KeepNaNs: true, }, }) diff --git a/src/query/block/meta.go b/src/query/block/meta.go index 85bdc98347..0e58ad24b3 100644 --- a/src/query/block/meta.go +++ b/src/query/block/meta.go @@ -68,6 +68,8 @@ type ResultMetadata struct { Warnings Warnings // Resolutions is a list of resolutions for series obtained by this query. Resolutions []time.Duration + // KeepNaNs indicates if NaNs should be kept when returning results. + KeepNaNs bool } // NewResultMetadata creates a new result metadata. diff --git a/src/query/functions/aggregation/take.go b/src/query/functions/aggregation/take.go index eab7ceee2c..098cea41d9 100644 --- a/src/query/functions/aggregation/take.go +++ b/src/query/functions/aggregation/take.go @@ -39,39 +39,36 @@ const ( TopKType = "topk" ) -type takeFunc func(values []float64, buckets [][]int) []float64 +type valueAndMeta struct { + val float64 + seriesMeta block.SeriesMeta +} + +type takeFunc func(heap utils.FloatHeap, values []float64, buckets [][]int) []float64 +type takeInstantFunc func(heap utils.FloatHeap, values []float64, buckets [][]int, seriesMetas []block.SeriesMeta) []valueAndMeta // NewTakeOp creates a new takeK operation func NewTakeOp( opType string, params NodeParams, ) (parser.Params, error) { - takeTop := opType == TopKType - if !takeTop && opType != BottomKType { - return baseOp{}, fmt.Errorf("operator not supported: %s", opType) - } - - var fn takeFunc k := int(params.Parameter) - if k < 1 { - fn = func(values []float64, buckets [][]int) []float64 { - return takeNone(values, buckets) - } - } else { - heap := utils.NewFloatHeap(takeTop, k) - fn = func(values []float64, buckets [][]int) []float64 { - return takeFn(heap, values, buckets) - } + fn := func(heap utils.FloatHeap, values []float64, buckets [][]int) []float64 { + return takeFn(heap, values, buckets) } - - return newTakeOp(params, opType, fn), nil + fnInstant := func(heap utils.FloatHeap, values []float64, buckets [][]int, seriesMetas []block.SeriesMeta) []valueAndMeta { + return takeInstantFn(heap, values, buckets, seriesMetas) + } + return newTakeOp(params, opType, k, fn, fnInstant), nil } // takeOp stores required properties for take ops type takeOp struct { - params NodeParams - opType string - takeFunc takeFunc + params NodeParams + opType string + k int + takeFunc takeFunc + takeInstantFunc takeInstantFunc } // OpType for the operator @@ -95,11 +92,13 @@ func (o takeOp) Node( } } -func newTakeOp(params NodeParams, opType string, takeFunc takeFunc) takeOp { +func newTakeOp(params NodeParams, opType string, k int, takeFunc takeFunc, takeInstantFunc takeInstantFunc) takeOp { return takeOp{ - params: params, - opType: opType, - takeFunc: takeFunc, + params: params, + opType: opType, + k: k, + takeFunc: takeFunc, + takeInstantFunc: takeInstantFunc, } } @@ -126,6 +125,12 @@ func (n *takeNode) ProcessBlock(queryCtx *models.QueryContext, ID parser.NodeID, return nil, err } + instantaneous := queryCtx.Options.Instantaneous + takeTop := n.op.opType == TopKType + if !takeTop && n.op.opType != BottomKType { + return nil, fmt.Errorf("operator not supported: %s", n.op.opType) + } + params := n.op.params meta := b.Meta() seriesMetas := utils.FlattenMetadata(meta, stepIter.SeriesMeta()) @@ -136,8 +141,23 @@ func (n *takeNode) ProcessBlock(queryCtx *models.QueryContext, ID parser.NodeID, seriesMetas, ) - // retain original metadatas - builder, err := n.controller.BlockBuilder(queryCtx, meta, stepIter.SeriesMeta()) + seriesCount := maxSeriesCount(buckets) + if instantaneous { + heapSize := seriesCount + if n.op.k < seriesCount { + heapSize = n.op.k + } + + heap := utils.NewFloatHeap(takeTop, heapSize) + return n.processBlockInstantaneous(heap, queryCtx, meta, stepIter, seriesMetas, buckets) + } + + if n.op.k >= seriesCount { + return b, nil + } + + heap := utils.NewFloatHeap(takeTop, n.op.k) + builder, err := n.controller.BlockBuilder(queryCtx, meta, seriesMetas) if err != nil { return nil, err } @@ -147,34 +167,95 @@ func (n *takeNode) ProcessBlock(queryCtx *models.QueryContext, ID parser.NodeID, } for index := 0; stepIter.Next(); index++ { - step := stepIter.Current() - values := step.Values() - aggregatedValues := n.op.takeFunc(values, buckets) - if err := builder.AppendValues(index, aggregatedValues); err != nil { + values := stepIter.Current().Values() + if err := builder.AppendValues(index, n.op.takeFunc(heap, values, buckets)); err != nil { return nil, err } } - if err = stepIter.Err(); err != nil { return nil, err } - return builder.Build(), nil } -// shortcut to return empty when taking <= 0 values -func takeNone(values []float64, buckets [][]int) []float64 { - util.Memset(values, math.NaN()) - return values +func maxSeriesCount(buckets [][]int) int { + result := 0 + + for _, bucket := range buckets { + if len(bucket) > result { + result = len(bucket) + } + } + + return result +} + +func (n *takeNode) processBlockInstantaneous( + heap utils.FloatHeap, + queryCtx *models.QueryContext, + metadata block.Metadata, + stepIter block.StepIter, + seriesMetas []block.SeriesMeta, + buckets [][]int) (block.Block, error) { + ixLastStep := stepIter.StepCount() - 1 //we only care for the last step values for the instant query + for i := 0; i <= ixLastStep; i++ { + if !stepIter.Next() { + return nil, fmt.Errorf("invalid step count; expected %d got %d", stepIter.StepCount(), i+1) + } + } + metadata.ResultMetadata.KeepNaNs = true + values := stepIter.Current().Values() + takenSortedValues := n.op.takeInstantFunc(heap, values, buckets, seriesMetas) + blockValues, blockSeries := mapToValuesAndSeriesMetas(takenSortedValues) + + //adjust bounds to contain single step + time, err := metadata.Bounds.TimeForIndex(ixLastStep) + if err != nil { + return nil, err + } + metadata.Bounds = models.Bounds{ + Start: time, + Duration: metadata.Bounds.StepSize, + StepSize: metadata.Bounds.StepSize, + } + + blockBuilder, err := n.controller.BlockBuilder(queryCtx, metadata, blockSeries) + if err != nil { + return nil, err + } + if err = blockBuilder.AddCols(1); err != nil { + return nil, err + } + if err := blockBuilder.AppendValues(0, blockValues); err != nil { + return nil, err + } + if err = stepIter.Err(); err != nil { + return nil, err + } + return blockBuilder.Build(), nil +} + +func mapToValuesAndSeriesMetas(takenSortedValues []valueAndMeta) ([]float64, []block.SeriesMeta) { + blockValues := make([]float64, 0, len(takenSortedValues)) + blockSeries := make([]block.SeriesMeta, 0, len(takenSortedValues)) + for _, sortedValue := range takenSortedValues { + blockValues = append(blockValues, sortedValue.val) + blockSeries = append(blockSeries, sortedValue.seriesMeta) + } + return blockValues, blockSeries } func takeFn(heap utils.FloatHeap, values []float64, buckets [][]int) []float64 { - cap := heap.Cap() + capacity := heap.Cap() + if capacity < 1 { + util.Memset(values, math.NaN()) + return values + } for _, bucket := range buckets { // If this bucket's length is less than or equal to the heap's // capacity do not need to clear any values from the input vector, // as they are all included in the output. - if len(bucket) <= cap { + if len(bucket) <= capacity { continue } @@ -198,3 +279,28 @@ func takeFn(heap utils.FloatHeap, values []float64, buckets [][]int) []float64 { return values } + +func takeInstantFn(heap utils.FloatHeap, values []float64, buckets [][]int, metas []block.SeriesMeta) []valueAndMeta { + var result = make([]valueAndMeta, 0, heap.Cap()) + if heap.Cap() < 1 { + return result + } + for _, bucket := range buckets { + for _, idx := range bucket { + val := values[idx] + heap.Push(val, idx) + } + + valIndexPairs := heap.OrderedFlush() + for _, pair := range valIndexPairs { + prevIndex := pair.Index + prevMeta := metas[prevIndex] + + result = append(result, valueAndMeta{ + val: pair.Val, + seriesMeta: prevMeta, + }) + } + } + return result +} diff --git a/src/query/functions/aggregation/take_test.go b/src/query/functions/aggregation/take_test.go index c0a5bef13b..854879062c 100644 --- a/src/query/functions/aggregation/take_test.go +++ b/src/query/functions/aggregation/take_test.go @@ -21,9 +21,11 @@ package aggregation import ( + "fmt" "math" "testing" + "github.com/m3db/m3/src/query/block" "github.com/m3db/m3/src/query/executor/transform" "github.com/m3db/m3/src/query/functions/utils" "github.com/m3db/m3/src/query/models" @@ -35,6 +37,66 @@ import ( "github.com/stretchr/testify/require" ) +func TestTakeInstantFn(t *testing.T) { + valuesMin := []float64{1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1} + buckets := [][]int{{0, 1, 2, 3}, {4}, {5, 6, 7, 8}} + + var ( + seriesMetasTakeOrdered = []block.SeriesMeta{ + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "0"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "1"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "2"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "0"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "1"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "0"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "1"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "0"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "1"}, {N: "group", V: "canary"}})}, + } + ) + + expectedMin := []valueAndMeta{ + {val: 1.1, seriesMeta: seriesMetasTakeOrdered[0]}, + {val: 2.1, seriesMeta: seriesMetasTakeOrdered[1]}, + {val: 3.1, seriesMeta: seriesMetasTakeOrdered[2]}, + + {val: 5.1, seriesMeta: seriesMetasTakeOrdered[4]}, + + {val: 6.1, seriesMeta: seriesMetasTakeOrdered[5]}, + {val: 7.1, seriesMeta: seriesMetasTakeOrdered[6]}, + {val: 8.1, seriesMeta: seriesMetasTakeOrdered[7]}, + } + + size := 3 + minHeap := utils.NewFloatHeap(false, size) + actual := takeInstantFn(minHeap, valuesMin, buckets, seriesMetasTakeOrdered) //9 + + actualString := fmt.Sprint(actual) + expectedString := fmt.Sprint(expectedMin) + + assert.EqualValues(t, expectedString, actualString) + + valuesMax := []float64{1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1} + expectedMax := []valueAndMeta{ + {val: 4.1, seriesMeta: seriesMetasTakeOrdered[3]}, + {val: 3.1, seriesMeta: seriesMetasTakeOrdered[2]}, + {val: 2.1, seriesMeta: seriesMetasTakeOrdered[1]}, + + {val: 5.1, seriesMeta: seriesMetasTakeOrdered[4]}, + + {val: 9.1, seriesMeta: seriesMetasTakeOrdered[8]}, + {val: 8.1, seriesMeta: seriesMetasTakeOrdered[7]}, + {val: 7.1, seriesMeta: seriesMetasTakeOrdered[6]}, + } + + maxHeap := utils.NewFloatHeap(true, size) + actual = takeInstantFn(maxHeap, valuesMax, buckets, seriesMetasTakeOrdered) + actualString = fmt.Sprint(actual) + expectedString = fmt.Sprint(expectedMax) + + assert.EqualValues(t, expectedString, actualString) +} + func TestTakeFn(t *testing.T) { valuesMin := []float64{1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1} buckets := [][]int{{0, 1, 2, 3}, {4}, {5, 6, 7}} @@ -98,6 +160,7 @@ func TestTakeTopFunctionFilteringWithoutA(t *testing.T) { }) require.NoError(t, err) sink := processTakeOp(t, op) + expected := [][]float64{ // Taking bottomk(1) of first two series, keeping both series {0, math.NaN(), math.NaN(), math.NaN(), math.NaN()}, @@ -139,3 +202,12 @@ func TestTakeTopFunctionFilteringWithoutALessThanOne(t *testing.T) { test.EqualsWithNansWithDelta(t, expected, sink.Values, math.Pow10(-5)) assert.Equal(t, bounds, sink.Meta.Bounds) } + +func TestTakeOpParamIsNaN(t *testing.T) { + op, err := NewTakeOp(TopKType, NodeParams{ + Parameter: math.NaN(), + }) + require.NoError(t, err) + assert.True(t, op.(takeOp).k < 0) +} + diff --git a/src/query/functions/linear/sort.go b/src/query/functions/linear/sort.go index b0f7b88665..9d696b9d04 100644 --- a/src/query/functions/linear/sort.go +++ b/src/query/functions/linear/sort.go @@ -20,13 +20,148 @@ package linear -const ( - // NB: Because Prometheus's sort and sort_desc only look at the last value, - // these functions are essentially noops in M3 as we don't support instant queries. +import ( + "fmt" + "sort" + + "github.com/m3db/m3/src/query/block" + "github.com/m3db/m3/src/query/executor/transform" + "github.com/m3db/m3/src/query/functions/utils" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/parser" +) +const ( // SortType returns timeseries elements sorted by their values, in ascending order. SortType = "sort" // SortDescType is the same as sort, but sorts in descending order. SortDescType = "sort_desc" ) + +type sortOp struct { + opType string + lessFn lessFn +} + +// OpType for the operator +func (o sortOp) OpType() string { + return o.opType +} + +// String representation +func (o sortOp) String() string { + return fmt.Sprintf("type: %s", o.opType) +} + +type sortNode struct { + op sortOp + controller *transform.Controller +} + +type valueAndMeta struct { + val float64 + seriesMeta block.SeriesMeta +} + +type lessFn func (i, j float64) bool + +// Node creates an execution node +func (o sortOp) Node( + controller *transform.Controller, + _ transform.Options, +) transform.OpNode { + return &sortNode{ + op: o, + controller: controller, + } +} + +func (n *sortNode) Params() parser.Params { + return n.op +} + +func (n *sortNode) Process(queryCtx *models.QueryContext, ID parser.NodeID, b block.Block) error { + return transform.ProcessSimpleBlock(n, n.controller, queryCtx, ID, b) +} + +func (n *sortNode) ProcessBlock(queryCtx *models.QueryContext, ID parser.NodeID, b block.Block) (block.Block, error) { + if !queryCtx.Options.Instantaneous { + return b, nil + } + stepIter, err := b.StepIter() + if err != nil { + return nil, err + } + + meta := b.Meta() + seriesMetas := utils.FlattenMetadata(meta, stepIter.SeriesMeta()) + return n.processInstantBlock(queryCtx, stepIter, meta, seriesMetas) +} + +func (n *sortNode) processInstantBlock(queryCtx *models.QueryContext, stepIter block.StepIter, meta block.Metadata, seriesMetas []block.SeriesMeta) (block.Block, error) { + ixLastStep := stepIter.StepCount() - 1 //we only care for the last step values for the instant query + for i := 0; i <= ixLastStep; i++ { + if !stepIter.Next() { + return nil, fmt.Errorf("invalid step count; expected %d got %d", stepIter.StepCount(), i+1) + } + } + values := stepIter.Current().Values() + meta.ResultMetadata.KeepNaNs = true + valuesToSort := make([]valueAndMeta, len(values)) + for i, value := range values { + valuesToSort[i] = valueAndMeta{ + val: value, + seriesMeta: seriesMetas[i], + } + } + + sort.Slice(valuesToSort, func(i, j int) bool { + return n.op.lessFn(valuesToSort[i].val, valuesToSort[j].val) + }) + + for i, sorted := range valuesToSort { + values[i] = sorted.val + seriesMetas[i] = sorted.seriesMeta + } + + //adjust bounds to contain single step + time, err := meta.Bounds.TimeForIndex(ixLastStep) + if err != nil { + return nil, err + } + meta.Bounds = models.Bounds{ + Start: time, + Duration: meta.Bounds.StepSize, + StepSize: meta.Bounds.StepSize, + } + + blockBuilder, err := n.controller.BlockBuilder(queryCtx, meta, seriesMetas) + if err != nil { + return nil, err + } + if err = blockBuilder.AddCols(1); err != nil { + return nil, err + } + if err := blockBuilder.AppendValues(0, values); err != nil { + return nil, err + } + if err = stepIter.Err(); err != nil { + return nil, err + } + return blockBuilder.Build(), nil +} + +func NewSortOp(opType string) (parser.Params, error) { + ascending := opType == SortType + if !ascending && opType != SortDescType { + return nil, fmt.Errorf("operator not supported: %s", opType) + } + + lessFn := utils.GreaterWithNaNs + if ascending { + lessFn = utils.LesserWithNaNs + } + + return sortOp{opType, lessFn}, nil +} diff --git a/src/query/functions/linear/sort_test.go b/src/query/functions/linear/sort_test.go new file mode 100644 index 0000000000..e1a4d46b3e --- /dev/null +++ b/src/query/functions/linear/sort_test.go @@ -0,0 +1,141 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package linear + +import ( + "math" + "testing" + "time" + + "github.com/m3db/m3/src/query/block" + "github.com/m3db/m3/src/query/executor/transform" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/parser" + "github.com/m3db/m3/src/query/test" + "github.com/m3db/m3/src/query/test/executor" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestShouldFailWhenOpTypeIsInvalid(t *testing.T) { + _, err := NewSortOp("sortAsc") + require.Error(t, err) +} + +func TestSortAscInstant(t *testing.T) { + op, err := NewSortOp(SortType) + require.NoError(t, err) + + sink := processSortOp(t, op, true) + + expected := [][]float64{ + {100}, + {200}, + {300}, + {400}, + {500}, + {600}, + {700}, + {800}, + {math.NaN()}, + } + + assert.Equal(t, seriesMetas, sink.Metas) + test.EqualsWithNansWithDelta(t, expected, sink.Values, math.Pow10(-5)) +} + +func TestSortDescInstant(t *testing.T) { + op, err := NewSortOp(SortDescType) + require.NoError(t, err) + + sink := processSortOp(t, op, true) + + expected := [][]float64{ + {800}, + {700}, + {600}, + {500}, + {400}, + {300}, + {200}, + {100}, + {math.NaN()}, + } + + assert.Equal(t, seriesMetas, sink.Metas) + test.EqualsWithNansWithDelta(t, expected, sink.Values, math.Pow10(-5)) +} + +func TestSortNop(t *testing.T) { + op, err := NewSortOp(SortDescType) + require.NoError(t, err) + + sink := processSortOp(t, op, false) + + expected := v + + assert.Equal(t, seriesMetas, sink.Metas) + test.EqualsWithNansWithDelta(t, expected, sink.Values, math.Pow10(-5)) +} + +var ( + seriesMetas = []block.SeriesMeta{ + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "0"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "1"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "2"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "0"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "api-server"}, {N: "instance", V: "1"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "0"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "1"}, {N: "group", V: "production"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "0"}, {N: "group", V: "canary"}})}, + {Tags: test.StringTagsToTags(test.StringTags{{N: "job", V: "app-server"}, {N: "instance", V: "1"}, {N: "group", V: "canary"}})}, + } + + v = [][]float64{ + {60, 70, 80, 90, 100}, + {150, 160, 170, 180, 200}, + {180, 210, 240, 270, 300}, + {240, 280, 320, 360, 400}, + {math.NaN(), math.NaN(), math.NaN(), math.NaN(), math.NaN()}, + {300, 350, 400, 450, 500}, + {360, 420, 480, 540, 600}, + {320, 390, 460, 530, 700}, + {480, 560, 640, 720, 800}, + } + + bounds = models.Bounds{ + Start: time.Now(), + Duration: time.Minute * 5, + StepSize: time.Minute, + } +) + +func processSortOp(t *testing.T, op parser.Params, instant bool) *executor.SinkNode { + bl := test.NewBlockFromValuesWithSeriesMeta(bounds, seriesMetas, v) + c, sink := executor.NewControllerWithSink(parser.NodeID(1)) + node := op.(sortOp).Node(c, transform.Options{}) + queryContext := models.NoopQueryContext() + queryContext.Options.Instantaneous = instant + err := node.Process(queryContext, parser.NodeID(0), bl) + require.NoError(t, err) + return sink +} diff --git a/src/query/functions/utils/heap.go b/src/query/functions/utils/heap.go index 1e3078025a..1587a1da00 100644 --- a/src/query/functions/utils/heap.go +++ b/src/query/functions/utils/heap.go @@ -22,6 +22,8 @@ package utils import ( "container/heap" + "math" + "sort" ) // ValueIndexPair is a pair of float value and index at which it exists @@ -33,17 +35,38 @@ type ValueIndexPair struct { type lessFn func(ValueIndexPair, ValueIndexPair) bool func maxHeapLess(i, j ValueIndexPair) bool { - if i.Val == j.Val { + if equalWithNaNs(i.Val, j.Val) { return i.Index > j.Index } - return i.Val < j.Val + return i.Val < j.Val || lesserIfNaNs(i.Val, j.Val) } func minHeapLess(i, j ValueIndexPair) bool { - if i.Val == j.Val { + if equalWithNaNs(i.Val, j.Val) { return i.Index > j.Index } - return i.Val > j.Val + return i.Val > j.Val || lesserIfNaNs(i.Val, j.Val) +} + +// Compares two floats for equality with NaNs taken into account. +func equalWithNaNs(i,j float64) bool { + return i == j || math.IsNaN(i) && math.IsNaN(j) +} + +// Compares NaNs. +// Basically, we do not want to add NaNs to the heap when it has reached it's cap so this fn should be used to prevent this. +func lesserIfNaNs(i,j float64) bool { + return math.IsNaN(i) && !math.IsNaN(j) +} + +// Compares two float64 values which one is lesser with NaNs. NaNs are always sorted away. +func LesserWithNaNs(i, j float64) bool { + return i < j || math.IsNaN(j) && !math.IsNaN(i) +} + +// Compares two float64 values which one is greater with NaNs. NaNs are always sorted away. +func GreaterWithNaNs(i, j float64) bool { + return i > j || math.IsNaN(j) && !math.IsNaN(i) } // FloatHeap is a heap that can be given a maximum size @@ -82,7 +105,7 @@ func NewFloatHeap(isMaxHeap bool, capacity int) FloatHeap { } // Push pushes a value and index pair to the heap -func (fh FloatHeap) Push(value float64, index int) { +func (fh *FloatHeap) Push(value float64, index int) { h := fh.floatHeap // If capacity is zero or negative, allow infinite size heap if fh.capacity > 0 { @@ -99,8 +122,8 @@ func (fh FloatHeap) Push(value float64, index int) { // NB(arnikola): unfortunately, can't just replace first // element as it may not respect internal order. Need to // run heap.Fix() to rectify this - if fh.isMaxHeap && value > peek.Val || - (!fh.isMaxHeap && value < peek.Val) { + if (fh.isMaxHeap && GreaterWithNaNs(value, peek.Val)) || + (!fh.isMaxHeap && LesserWithNaNs(value, peek.Val)) { h.heap[0] = ValueIndexPair{ Val: value, Index: index, @@ -122,12 +145,12 @@ func (fh FloatHeap) Push(value float64, index int) { } // Len returns the current length of the heap -func (fh FloatHeap) Len() int { +func (fh *FloatHeap) Len() int { return fh.floatHeap.Len() } // Cap returns the capacity of the heap -func (fh FloatHeap) Cap() int { +func (fh *FloatHeap) Cap() int { return fh.capacity } @@ -137,14 +160,23 @@ func (fh *FloatHeap) Reset() { } // Flush flushes the float heap and resets it. Does not guarantee order. -func (fh FloatHeap) Flush() []ValueIndexPair { +func (fh *FloatHeap) Flush() []ValueIndexPair { values := fh.floatHeap.heap fh.Reset() return values } +// OrderedFlush flushes the float heap and returns values in order. +func (fh *FloatHeap) OrderedFlush() []ValueIndexPair { + flushed := fh.Flush() + sort.Slice(flushed, func(i, j int) bool { + return !fh.floatHeap.less(flushed[i], flushed[j]) //reverse sort + }) + return flushed +} + // Peek reveals the top value of the heap without mutating the heap. -func (fh FloatHeap) Peek() (ValueIndexPair, bool) { +func (fh *FloatHeap) Peek() (ValueIndexPair, bool) { h := fh.floatHeap.heap if len(h) == 0 { return ValueIndexPair{}, false diff --git a/src/query/functions/utils/heap_test.go b/src/query/functions/utils/heap_test.go index c8a8e0f842..79e2a305fa 100644 --- a/src/query/functions/utils/heap_test.go +++ b/src/query/functions/utils/heap_test.go @@ -21,10 +21,13 @@ package utils import ( + "math" "math/rand" "sort" "testing" + "github.com/m3db/m3/src/query/test" + "github.com/stretchr/testify/assert" ) @@ -216,3 +219,135 @@ func TestNegativeCapacityHeap(t *testing.T) { assert.Equal(t, testArray[pair.Index], pair.Val) } } + +func equalPairs(t *testing.T, expected, actual []ValueIndexPair) { + assert.Equal(t, len(expected), len(actual)) + for i, e := range expected { + test.EqualsWithNans(t, e.Val, actual[i].Val) + assert.Equal(t, e.Index, actual[i].Index) + } +} + +func TestFlushOrdered(t *testing.T) { + maxHeap := NewFloatHeap(true, 3) + + maxHeap.Push(0.1, 0) + maxHeap.Push(1.1, 1) + maxHeap.Push(2.1, 2) + maxHeap.Push(3.1, 3) + + actualMax := maxHeap.OrderedFlush() + + assert.Equal(t, []ValueIndexPair{ + {Val: 3.1, Index: 3}, + {Val: 2.1, Index: 2}, + {Val: 1.1, Index: 1}, + }, actualMax) + assert.Equal(t, 0, maxHeap.Len()) + + minHeap := NewFloatHeap(false, 3) + minHeap.Push(0.1, 0) + minHeap.Push(1.1, 1) + minHeap.Push(2.1, 2) + minHeap.Push(3.1, 3) + + actualMin := minHeap.OrderedFlush() + + assert.Equal(t, []ValueIndexPair{ + {Val: 0.1, Index: 0}, + {Val: 1.1, Index: 1}, + {Val: 2.1, Index: 2}, + }, actualMin) + assert.Equal(t, 0, minHeap.Len()) +} + +func TestFlushOrderedWhenRandomInsertionOrder(t *testing.T) { + maxHeap := NewFloatHeap(true, 3) + + maxHeap.Push(math.NaN(), 4) + maxHeap.Push(0.1, 0) + maxHeap.Push(2.1, 2) + maxHeap.Push(1.1, 1) + maxHeap.Push(3.1, 3) + maxHeap.Push(math.NaN(), 5) + + actualMax := maxHeap.OrderedFlush() + + assert.Equal(t, []ValueIndexPair{ + {Val: 3.1, Index: 3}, + {Val: 2.1, Index: 2}, + {Val: 1.1, Index: 1}, + }, actualMax) + assert.Equal(t, 0, maxHeap.Len()) + + minHeap := NewFloatHeap(false, 3) + maxHeap.Push(math.NaN(), 4) + minHeap.Push(0.1, 0) + minHeap.Push(2.1, 2) + minHeap.Push(1.1, 1) + minHeap.Push(3.1, 3) + maxHeap.Push(math.NaN(), 5) + + actualMin := minHeap.OrderedFlush() + + assert.Equal(t, []ValueIndexPair{ + {Val: 0.1, Index: 0}, + {Val: 1.1, Index: 1}, + {Val: 2.1, Index: 2}, + }, actualMin) + assert.Equal(t, 0, minHeap.Len()) +} + +func TestFlushOrderedWhenRandomInsertionOrderAndTakeNaNs(t *testing.T) { + maxHeap := NewFloatHeap(true, 3) + maxHeap.Push(math.NaN(), 4) + maxHeap.Push(1.1, 1) + maxHeap.Push(3.1, 3) + maxHeap.Push(math.NaN(), 5) + + actualMax := maxHeap.OrderedFlush() + + equalPairs(t, []ValueIndexPair{ + {Val: 3.1, Index: 3}, + {Val: 1.1, Index: 1}, + {Val: math.NaN(), Index: 4}, + }, actualMax) + assert.Equal(t, 0, maxHeap.Len()) + + minHeap := NewFloatHeap(false, 3) + minHeap.Push(math.NaN(), 4) + minHeap.Push(0.1, 0) + minHeap.Push(2.1, 2) + minHeap.Push(math.NaN(), 5) + + actualMin := minHeap.OrderedFlush() + + equalPairs(t, []ValueIndexPair{ + {Val: 0.1, Index: 0}, + {Val: 2.1, Index: 2}, + {Val: math.NaN(), Index: 4}, + }, actualMin) + assert.Equal(t, 0, minHeap.Len()) +} + +func TestSortLesserWithNaNs(t *testing.T) { + actual := []float64{ 5.0, 4.1, math.NaN(), 8.6, 0.1 } + expected := []float64{ 0.1, 4.1, 5.0, 8.6, math.NaN() } + + sort.Slice(actual, func(i, j int) bool { + return LesserWithNaNs(actual[i], actual[j]) + }) + + test.EqualsWithNans(t, expected, actual) +} + +func TestSortGreaterWithNaNs(t *testing.T) { + actual := []float64{ 5.0, 4.1, math.NaN(), 8.6, 0.1 } + expected := []float64{ 8.6, 5.0, 4.1, 0.1, math.NaN() } + + sort.Slice(actual, func(i, j int) bool { + return GreaterWithNaNs(actual[i], actual[j]) + }) + + test.EqualsWithNans(t, expected, actual) +} diff --git a/src/query/models/query_context.go b/src/query/models/query_context.go index 0da1e3daf5..3522b05833 100644 --- a/src/query/models/query_context.go +++ b/src/query/models/query_context.go @@ -44,6 +44,9 @@ type QueryContextOptions struct { LimitMaxDocs int // RequireExhaustive results in an error if the query exceeds the series limit. RequireExhaustive bool + // Instantaneous indicates an instant query. + Instantaneous bool + // RestrictFetchType restricts the query fetches. RestrictFetchType *RestrictFetchTypeQueryContextOptions } diff --git a/src/query/parser/promql/matchers.go b/src/query/parser/promql/matchers.go index 50b965761c..c25e618a61 100644 --- a/src/query/parser/promql/matchers.go +++ b/src/query/parser/promql/matchers.go @@ -295,10 +295,11 @@ func NewFunctionExpr( p, err = scalar.NewTimeOp(tagOptions) return p, true, err - // NB: no-ops. case linear.SortType, linear.SortDescType: - return nil, false, err + p, err = linear.NewSortOp(name) + return p, true, err + // NB: no-ops. case scalar.ScalarType: return nil, false, err diff --git a/src/query/parser/promql/parse_test.go b/src/query/parser/promql/parse_test.go index 656d578a05..9e7aef9ee6 100644 --- a/src/query/parser/promql/parse_test.go +++ b/src/query/parser/promql/parse_test.go @@ -303,10 +303,12 @@ func TestSort(t *testing.T) { require.NoError(t, err) transforms, edges, err := p.DAG() require.NoError(t, err) - assert.Len(t, transforms, 1) + assert.Len(t, transforms, 2) assert.Equal(t, transforms[0].Op.OpType(), functions.FetchType) assert.Equal(t, transforms[0].ID, parser.NodeID("0")) - assert.Len(t, edges, 0) + assert.Equal(t, transforms[1].Op.OpType(), tt.expectedType) + assert.Equal(t, transforms[1].ID, parser.NodeID("1")) + assert.Len(t, edges, 1) }) } } diff --git a/src/query/test/compatibility/testdata/aggregators.test b/src/query/test/compatibility/testdata/aggregators.test index 8eb9cbcc7e..ebcbcc9fd5 100644 --- a/src/query/test/compatibility/testdata/aggregators.test +++ b/src/query/test/compatibility/testdata/aggregators.test @@ -159,64 +159,63 @@ load 5m http_requests{job="app-server", instance="1", group="canary"} 0+80x10 foo 3+0x10 -# FAILING issue #12. All topk and bottomk tests are failing. -#eval_ordered instant at 50m topk(3, http_requests) -# http_requests{group="canary", instance="1", job="app-server"} 800 -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="production", instance="1", job="app-server"} 600 +eval_ordered instant at 50m topk(3, http_requests) + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="1", job="app-server"} 600 -#eval_ordered instant at 50m topk((3), (http_requests)) -# http_requests{group="canary", instance="1", job="app-server"} 800 -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="production", instance="1", job="app-server"} 600 +eval_ordered instant at 50m topk((3), (http_requests)) + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="1", job="app-server"} 600 -#eval_ordered instant at 50m topk(5, http_requests{group="canary",job="app-server"}) -# http_requests{group="canary", instance="1", job="app-server"} 800 -# http_requests{group="canary", instance="0", job="app-server"} 700 +eval_ordered instant at 50m topk(5, http_requests{group="canary",job="app-server"}) + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="0", job="app-server"} 700 -#eval_ordered instant at 50m bottomk(3, http_requests) -# http_requests{group="production", instance="0", job="api-server"} 100 -# http_requests{group="production", instance="1", job="api-server"} 200 -# http_requests{group="canary", instance="0", job="api-server"} 300 +eval_ordered instant at 50m bottomk(3, http_requests) + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="0", job="api-server"} 300 -#eval_ordered instant at 50m bottomk(5, http_requests{group="canary",job="app-server"}) -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="canary", instance="1", job="app-server"} 800 +eval_ordered instant at 50m bottomk(5, http_requests{group="canary",job="app-server"}) + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="1", job="app-server"} 800 eval instant at 50m topk by (group) (1, http_requests) http_requests{group="production", instance="1", job="app-server"} 600 http_requests{group="canary", instance="1", job="app-server"} 800 -#eval instant at 50m bottomk by (group) (2, http_requests) -# http_requests{group="canary", instance="0", job="api-server"} 300 -# http_requests{group="canary", instance="1", job="api-server"} 400 -# http_requests{group="production", instance="0", job="api-server"} 100 -# http_requests{group="production", instance="1", job="api-server"} 200 +eval instant at 50m bottomk by (group) (2, http_requests) + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="1", job="api-server"} 200 -#eval_ordered instant at 50m bottomk by (group) (2, http_requests{group="production"}) -# http_requests{group="production", instance="0", job="api-server"} 100 -# http_requests{group="production", instance="1", job="api-server"} 200 +eval_ordered instant at 50m bottomk by (group) (2, http_requests{group="production"}) + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="1", job="api-server"} 200 # Test NaN is sorted away from the top/bottom. -#eval_ordered instant at 50m topk(3, http_requests{job="api-server",group="production"}) -# http_requests{job="api-server", instance="1", group="production"} 200 -# http_requests{job="api-server", instance="0", group="production"} 100 -# http_requests{job="api-server", instance="2", group="production"} NaN +eval_ordered instant at 50m topk(3, http_requests{job="api-server",group="production"}) + http_requests{job="api-server", instance="1", group="production"} 200 + http_requests{job="api-server", instance="0", group="production"} 100 + http_requests{job="api-server", instance="2", group="production"} NaN -#eval_ordered instant at 50m bottomk(3, http_requests{job="api-server",group="production"}) -# http_requests{job="api-server", instance="0", group="production"} 100 -# http_requests{job="api-server", instance="1", group="production"} 200 -# http_requests{job="api-server", instance="2", group="production"} NaN +eval_ordered instant at 50m bottomk(3, http_requests{job="api-server",group="production"}) + http_requests{job="api-server", instance="0", group="production"} 100 + http_requests{job="api-server", instance="1", group="production"} 200 + http_requests{job="api-server", instance="2", group="production"} NaN # Test topk and bottomk allocate min(k, input_vector) for results vector -#eval_ordered instant at 50m bottomk(9999999999, http_requests{job="app-server",group="canary"}) -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="canary", instance="1", job="app-server"} 800 - -#eval_ordered instant at 50m topk(9999999999, http_requests{job="api-server",group="production"}) -# http_requests{job="api-server", instance="1", group="production"} 200 -# http_requests{job="api-server", instance="0", group="production"} 100 -# http_requests{job="api-server", instance="2", group="production"} NaN +eval_ordered instant at 50m bottomk(9999999999, http_requests{job="app-server",group="canary"}) + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="1", job="app-server"} 800 + +eval_ordered instant at 50m topk(9999999999, http_requests{job="api-server",group="production"}) + http_requests{job="api-server", instance="1", group="production"} 200 + http_requests{job="api-server", instance="0", group="production"} 100 + http_requests{job="api-server", instance="2", group="production"} NaN # Bug #5276. #eval_ordered instant at 50m topk(scalar(foo), http_requests) diff --git a/src/query/test/compatibility/testdata/functions.test b/src/query/test/compatibility/testdata/functions.test index e311bdc461..49de42bb1a 100644 --- a/src/query/test/compatibility/testdata/functions.test +++ b/src/query/test/compatibility/testdata/functions.test @@ -342,28 +342,27 @@ load 5m http_requests{job="app-server", instance="0", group="canary"} 0+70x10 http_requests{job="app-server", instance="1", group="canary"} 0+80x10 -# FAILING issue #28: -#eval_ordered instant at 50m sort(http_requests) -# http_requests{group="production", instance="0", job="api-server"} 100 -# http_requests{group="production", instance="1", job="api-server"} 200 -# http_requests{group="canary", instance="0", job="api-server"} 300 -# http_requests{group="canary", instance="1", job="api-server"} 400 -# http_requests{group="production", instance="0", job="app-server"} 500 -# http_requests{group="production", instance="1", job="app-server"} 600 -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="canary", instance="1", job="app-server"} 800 -# http_requests{group="canary", instance="2", job="api-server"} NaN - -#eval_ordered instant at 50m sort_desc(http_requests) -# http_requests{group="canary", instance="1", job="app-server"} 800 -# http_requests{group="canary", instance="0", job="app-server"} 700 -# http_requests{group="production", instance="1", job="app-server"} 600 -# http_requests{group="production", instance="0", job="app-server"} 500 -# http_requests{group="canary", instance="1", job="api-server"} 400 -# http_requests{group="canary", instance="0", job="api-server"} 300 -# http_requests{group="production", instance="1", job="api-server"} 200 -# http_requests{group="production", instance="0", job="api-server"} 100 -# http_requests{group="canary", instance="2", job="api-server"} NaN +eval_ordered instant at 50m sort(http_requests) + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="2", job="api-server"} NaN + +eval_ordered instant at 50m sort_desc(http_requests) + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="canary", instance="2", job="api-server"} NaN # Tests for holt_winters clear From 5a3a12bba59e058b7a4cee6ec472c3f5786f741b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Thu, 5 Nov 2020 18:02:20 +0200 Subject: [PATCH 25/47] [build] Enable golangci-lint (#2766) --- .buildkite/pipeline.yml | 13 +- .excludemetalint | 11 - .golangci.yml | 284 +++++++++++++ .metalinter.json | 40 -- Makefile | 42 +- go.mod | 19 +- go.sum | 61 ++- scripts/install-gometalinter.sh | 391 ------------------ scripts/run-ci-lint.sh | 18 + src/cmd/tools/linter/gorules/rules.go | 29 ++ src/cmd/tools/linter/main/main.go | 29 ++ .../api/v1/handler/placement/set_test.go | 4 +- src/x/instrument/methods.go | 2 +- tools.go | 2 - 14 files changed, 460 insertions(+), 485 deletions(-) delete mode 100644 .excludemetalint create mode 100644 .golangci.yml delete mode 100644 .metalinter.json delete mode 100755 scripts/install-gometalinter.sh create mode 100755 scripts/run-ci-lint.sh create mode 100644 src/cmd/tools/linter/gorules/rules.go create mode 100644 src/cmd/tools/linter/main/main.go diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c71d836da5..b435db97ea 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -35,8 +35,17 @@ steps: run: app workdir: /go/src/github.com/m3db/m3 <<: *common - - name: "Services, Tools, Metalint" - command: make clean install-vendor-m3 services tools metalint + - name: "Services, Tools" + command: make clean install-vendor-m3 services tools + plugins: + docker-compose#v2.5.1: + run: app + workdir: /go/src/github.com/m3db/m3 + <<: *common + - name: "Lint" + command: make clean lint + env: + CGO_ENABLED: 0 plugins: docker-compose#v2.5.1: run: app diff --git a/.excludemetalint b/.excludemetalint deleted file mode 100644 index 9cd47f901a..0000000000 --- a/.excludemetalint +++ /dev/null @@ -1,11 +0,0 @@ -.pb.go -_gen.go -_gen_test.go -_mock.go -_string.go -generated/ -mocks/ -vendor/ -src/m3ninx/x/bytes/slice_arraypool_gen.go -src/m3ninx/index/segment/mem/ids_map_gen.go -src/query/parser/m3ql/grammar.peg.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..57c383e62c --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,284 @@ +# options for analysis running +run: + # default concurrency is a available CPU number + # concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 10m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: [] + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + - generated/.* + # There is some very weird golangci-lint bug that causes analysis to fail on the + # eventlog_server_test.go file (perhaps caused by https://github.com/golangci/golangci-lint/issues/995). + # To avoid sporadic CI failures we exclude the file from analysis for the time being. + # In order to exclude the file from analysis, and not just any lints in it, we need + # to put the file in a separate directory since although golangci-lint skips issues + # from files in the skip-files list, it still runs analysis on them. + - eventlog/test$ + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: [] + + # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # modules-download-mode: readonly|release|vendor + modules-download-mode: readonly + + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + + +# all available settings of specific linters +linters-settings: + govet: + # report about shadowed variables + check-shadowing: true + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: github.com/m3db/m3 + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 10 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + depguard: + list-type: blacklist + include-go-root: false + packages: + - github.com/sirupsen/logrus + - github.com/golang/protobuf/jsonpb + - google.golang.org/protobuf/encoding/protojson + - github.com/golang/protobuf/proto + - google.golang.org/protobuf/proto + - github.com/tj/assert + packages-with-error-messages: + # specify an error message to output when a blacklisted package is used + github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + github.com/golang/protobuf/jsonpb: "replace with github.com/gogo/protobuf/jsonpb" + google.golang.org/protobuf/encoding/protojson: "replace with github.com/gogo/protobuf/jsonpb" + github.com/golang/protobuf/proto: "replace with github.com/gogo/protobuf/proto" + google.golang.org/protobuf/proto: "replace with github.com/gogo/protobuf/proto" + github.com/tj/assert: "use github.com/stretchr/testify/assert" + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + ignore-words: + - someword + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 100 + # tab width in spaces. Default to 1. + tab-width: 1 + unused: + # treat code as a program (not a library) and report unused exported identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unparam: + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + nakedret: + # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + max-func-lines: 30 + prealloc: + # XXX: we don't recommend using this linter before doing performance profiling. + # For most programs usage of prealloc will be a premature optimization. + + # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # True by default. + simple: true + range-loops: true # Report preallocation suggestions on range loops, true by default + for-loops: false # Report preallocation suggestions on for loops, false by default + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + + # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks. + # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". + enabled-tags: + - performance + enabled-checks: + - ruleguard + settings: # settings passed to gocritic + captLocal: # must be valid enabled check name + paramsOnly: true + rangeValCopy: + sizeThreshold: 32 + ruleguard: + rules: "src/cmd/tools/linter/gorules/rules.go" + gci: + # gci control golang package import order and make it always deterministic + local-prefixes: github.com/m3db/m3 + +linters: + enable: + - deadcode + - dogsled + - dupl + - errcheck + - exhaustive + - gci + - goconst + - gocritic + - gocyclo + - godox + - goimports + - golint + - gosimple + - govet + - ineffassign + - lll + - maligned + - megacheck + - misspell + - prealloc + - staticcheck + - structcheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - wsl + enable-all: false + disable: + - gomnd + - gochecknoinits + # Globals gonna global + - gochecknoglobals + # Overly harsh about long functions + - funlen + # Linter that checks that every comment ends in a period. + - godot + # Linter that makes you always use _test packages. + - testpackage + # Overly opinionated about how to construct errors + - goerr113 + # Noisy warnings about whether "nolint" directives are necessary + - nolintlint + # Deprecated project due to being prone to bad suggestions. + - interfacer + disable-all: false + presets: + # bodyclose, errcheck, gosec, govet, scopelint, staticcheck, typecheck + - bugs + # deadcode, ineffassign, structcheck, unparam, unused, varcheck + - unused + # gofmt, goimports + - format + # depguard, dupl, gochecknoglobals, gochecknoinits, goconst, gocritic, + # golint, gosimple, interfacer, lll, misspell, stylecheck, unconvert + - style + fast: false + + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + # Exclude table-driven tests from scopelint (https://github.com/golangci/golangci-lint/issues/281). + - "Using the variable on range scope `tt` in function literal" + - "Using the variable on range scope `test` in function literal" + # It's common to shadow `err` and rarely indicates a problems. See + # https://github.com/golang/go/issues/19490 for further details. + - 'shadow: declaration of "err" shadows declaration' + # We commonly expose profiling information on /debug/pprof so we need to disable the gosec + # lint for it. + - "Profiling endpoint is automatically exposed on /debug/pprof" + # We only use md5 for non-cryptographic purposes (e.g. generating ID's where we don't assume + # the ID's are cryptographicly secure). + - "Blacklisted import `crypto/md5`: weak cryptographic primitive" + # The logger is often our last option to communicate that an error occurred so if it returns + # an error we don't have an alternative to use. Since it's already unlikely that `Log` will + # return an error anyway we often skip checking the error for brevity. + - "Error return value of `\\(github.com\\/go-kit\\/kit\\/log.Logger\\).Log` is not checked" + # The caller is responsible for closing the Body of an `http.Response`. However, this step + # is usually performed in a defer function after the response has already been processed and + # so errors, which are already rare, can usually be safely ignored. + - "Error return value of `[a-zA-Z.]+.Body.Close` is not checked" + # The errcheck linter already checks for unhandled errors so we can disable the equivalent + # lint by gosec. + - "G104: Errors unhandled" + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + # Exclude some linters from running on tests files. + # - path: _test\.go + # linters: + # - gocyclo + # - errcheck + # - dupl + # - gosec + + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # Show only new issues created after git revision `REV` + new-from-rev: 57da65a558cc40f7de2f7f5c7528ca6c6278488e diff --git a/.metalinter.json b/.metalinter.json deleted file mode 100644 index 55125b9232..0000000000 --- a/.metalinter.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "Linters": { - "unused": { - "Command": "unused -tags 'integration big'", - "Pattern": "PATH:LINE:COL:MESSAGE" - }, - "gosimple": { - "Command": "gosimple -tags 'integration big'", - "Pattern": "PATH:LINE:COL:MESSAGE" - }, - "badtime": { - "Command": "badtime -tags 'integration big'", - "Pattern": "PATH:LINE:COL:MESSAGE" - }, - "varcheck": { - "Command": "varcheck -tags 'integration big'", - "Pattern": "PATH:LINE:COL:MESSAGE" - }, - "importorder": { - "Command": "importorder -patterns='STDLIB github.com/m3db EXTERNAL' -tags 'integration big'", - "Pattern": "PATH:LINE:MESSAGE" - } - }, - "Enable": [ - "varcheck", - "structcheck", - "goconst", - "ineffassign", - "unconvert", - "misspell", - "golint", - "unused", - "gosimple", - "badtime", - "maligned", - "importorder" - ], - "Deadline": "3m", - "EnableGC": true -} \ No newline at end of file diff --git a/Makefile b/Makefile index c737c4d4f8..f4e3531e55 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,6 @@ tools_bin_path := $(abspath ./_tools/bin) combined_bin_paths := $(tools_bin_path):$(gopath_bin_path) retool_src_prefix := $(m3_package_path)/_tools/src retool_package := github.com/twitchtv/retool -metalint_check := .ci/metalint.sh -metalint_config := .metalinter.json -metalint_exclude := .excludemetalint mocks_output_dir := generated/mocks mocks_rules_dir := generated/mocks proto_output_dir := generated/proto @@ -45,7 +42,6 @@ LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 $(GO_BUILD_COMMON_ENV) GO_RELEASER_DOCKER_IMAGE := goreleaser/goreleaser:v0.127.0 GO_RELEASER_RELEASE_ARGS ?= --rm-dist GO_RELEASER_WORKING_DIR := /go/src/github.com/m3db/m3 -GOMETALINT_VERSION := v2.0.5 # Retool will look for tools.json in the nearest parent git directory if not # explicitly told the current dir. Allow setting the base dir so that tools can @@ -93,6 +89,7 @@ TOOLS := \ verify_index_files \ carbon_load \ m3ctl \ + linter \ .PHONY: setup setup: @@ -168,7 +165,7 @@ tools-linux-amd64: $(LINUX_AMD64_ENV) make tools .PHONY: all -all: metalint test-ci-unit test-ci-integration services tools +all: lint test-ci-unit test-ci-integration services tools @echo Made all successfully .PHONY: install-tools @@ -177,10 +174,9 @@ install-tools: GOBIN=$(tools_bin_path) go install github.com/fossas/fossa-cli/cmd/fossa GOBIN=$(tools_bin_path) go install github.com/golang/mock/mockgen GOBIN=$(tools_bin_path) go install github.com/google/go-jsonnet/cmd/jsonnet - GOBIN=$(tools_bin_path) go install github.com/m3db/build-tools/linters/badtime - GOBIN=$(tools_bin_path) go install github.com/m3db/build-tools/linters/importorder GOBIN=$(tools_bin_path) go install github.com/m3db/build-tools/utilities/genclean GOBIN=$(tools_bin_path) go install github.com/m3db/tools/update-license + GOBIN=$(tools_bin_path) go install github.com/golangci/golangci-lint/cmd/golangci-lint GOBIN=$(tools_bin_path) go install github.com/mauricelam/genny GOBIN=$(tools_bin_path) go install github.com/mjibson/esc GOBIN=$(tools_bin_path) go install github.com/pointlander/peg @@ -189,11 +185,6 @@ install-tools: GOBIN=$(tools_bin_path) go install github.com/garethr/kubeval GOBIN=$(tools_bin_path) go install github.com/wjdp/htmltest -.PHONY: install-gometalinter -install-gometalinter: - @mkdir -p $(tools_bin_path) - ./scripts/install-gometalinter.sh -b $(tools_bin_path) -d $(GOMETALINT_VERSION) - .PHONY: check-for-goreleaser-github-token check-for-goreleaser-github-token: ifndef GITHUB_TOKEN @@ -265,8 +256,8 @@ SUBDIR_TARGETS := \ asset-gen \ genny-gen \ license-gen \ - all-gen \ - metalint + all-gen \ + lint .PHONY: test-ci-unit test-ci-unit: test-base @@ -374,11 +365,12 @@ test-ci-integration-$(SUBDIR): @echo "--- uploading coverage report" $(codecov_push) -f $(coverfile) -F $(SUBDIR) -.PHONY: metalint-$(SUBDIR) -metalint-$(SUBDIR): install-gometalinter install-linter-badtime install-linter-importorder - @echo "--- metalinting $(SUBDIR)" - @(PATH=$(combined_bin_paths):$(PATH) $(metalint_check) \ - $(metalint_config) $(metalint_exclude) src/$(SUBDIR)) +.PHONY: lint-$(SUBDIR) +lint-$(SUBDIR): export GO_BUILD_TAGS = $(GO_BUILD_TAGS_LIST) +lint-$(SUBDIR): install-tools linter + @echo "--- :golang: Running linters on $(SUBDIR)" + ./scripts/run-ci-lint.sh $(tools_bin_path)/golangci-lint ./src/$(SUBDIR)/... + ./bin/linter ./src/$(SUBDIR)/... endef @@ -392,9 +384,7 @@ endef # generate targets across SUBDIRS for each SUBDIR_TARGET. i.e. generate rules # which allow `make all-gen` to invoke `make all-gen-dbnode all-gen-coordinator ...` -# NB: we skip metalint explicity as the default target below requires less invocations -# of metalint and finishes faster. -$(foreach SUBDIR_TARGET, $(filter-out metalint,$(SUBDIR_TARGETS)), $(eval $(SUBDIR_TARGET_RULE))) +$(foreach SUBDIR_TARGET, $(SUBDIR_TARGETS), $(eval $(SUBDIR_TARGET_RULE))) # Builds the single kube bundle from individual manifest files. .PHONY: kube-gen-all @@ -411,7 +401,7 @@ go-mod-tidy: .PHONY: all-gen all-gen: \ install-tools \ - $(foreach SUBDIR_TARGET, $(filter-out metalint all-gen,$(SUBDIR_TARGETS)), $(SUBDIR_TARGET)) \ + $(foreach SUBDIR_TARGET, $(SUBDIR_TARGETS), $(SUBDIR_TARGET)) \ kube-gen-all \ go-mod-tidy @@ -462,12 +452,6 @@ else bash -c "source $(SELF_DIR)/.nvm/nvm.sh && nvm use 6 && $(node_cmd)" endif -.PHONY: metalint -metalint: install-gometalinter install-tools - @echo "--- metalinting src/" - @(PATH=$(tools_bin_path):$(PATH) $(metalint_check) \ - $(metalint_config) $(metalint_exclude) $(m3_package_path)/src/) - # Tests that all currently generated types match their contents if they were regenerated .PHONY: test-all-gen test-all-gen: all-gen diff --git a/go.mod b/go.mod index 91e3ee308a..8e46e640e9 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/davecgh/go-spew v1.1.1 github.com/docker/go-connections v0.4.0 // indirect + github.com/fatih/color v1.10.0 // indirect github.com/fortytw2/leaktest v1.2.1-0.20180901000122-b433bbd6d743 github.com/fossas/fossa-cli v1.0.30 github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94 @@ -34,9 +35,9 @@ require ( github.com/go-playground/universal-translator v0.17.0 // indirect github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.4.4 - github.com/golang/protobuf v1.3.3 + github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 - github.com/google/go-cmp v0.5.1 + github.com/google/go-cmp v0.5.2 github.com/google/go-jsonnet v0.16.0 github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f // indirect github.com/gorilla/handlers v1.4.2 // indirect @@ -47,6 +48,7 @@ require ( github.com/influxdata/influxdb v1.7.7 github.com/jhump/protoreflect v1.6.1 github.com/json-iterator/go v1.1.9 + github.com/kr/text v0.2.0 // indirect github.com/leanovate/gopter v0.2.8 github.com/lib/pq v1.6.0 // indirect github.com/lightstep/lightstep-tracer-go v0.18.1 @@ -65,6 +67,9 @@ require ( github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de github.com/mauricelam/genny v0.0.0-20180903214747-eb2c5232c885 github.com/mjibson/esc v0.1.0 + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/onsi/ginkgo v1.14.1 // indirect + github.com/onsi/gomega v1.10.2 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 @@ -90,8 +95,9 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/sergi/go-diff v1.1.0 github.com/shirou/gopsutil v2.20.5+incompatible // indirect + github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a // indirect - github.com/spf13/cobra v0.0.5 + github.com/spf13/cobra v1.1.1 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect github.com/stretchr/testify v1.6.1 @@ -116,8 +122,10 @@ require ( go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a + golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 + golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 google.golang.org/grpc v1.29.1 + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/go-ini/ini.v1 v1.57.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.7.0 @@ -125,8 +133,9 @@ require ( gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/validator.v2 v2.0.0-20160201165114-3e4f037f12a1 gopkg.in/vmihailenco/msgpack.v2 v2.8.3 - gopkg.in/yaml.v2 v2.2.8 + gopkg.in/yaml.v2 v2.3.0 gotest.tools v2.2.0+incompatible + honnef.co/go/tools v0.0.1-2020.1.6 // indirect ) // branch 0.9.3-pool-read-binary-3 diff --git a/go.sum b/go.sum index 027a861899..2971417542 100644 --- a/go.sum +++ b/go.sum @@ -160,7 +160,10 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -202,6 +205,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.2.1-0.20180901000122-b433bbd6d743 h1:QDM8xNoGxemDHdExynv+HzqkTPsFFZ8EyZdMwGElpGg= @@ -212,6 +217,8 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94 h1:NMtO+FvLt7roVanhHmJUsIRq9sEbEytH/PWNE+zR8vw= github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94/go.mod h1:L8VwozDBY4bGI25r29I6FURZus8xlVo/B7lNOSfre2g= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -311,6 +318,13 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -330,8 +344,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-jsonnet v0.16.0 h1:Nb4EEOp+rdeGGyB1rQ5eisgSAqrTnhf9ip+X6lzZbY0= @@ -500,6 +514,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leanovate/gopter v0.2.8 h1:eFPtJ3aa5zLfbxGROSNY75T9Dume60CWBAqoWQ3h/ig= github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= @@ -556,6 +572,8 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -611,6 +629,10 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -623,12 +645,20 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= +github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -763,6 +793,8 @@ github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvH github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -788,6 +820,8 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -798,6 +832,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= @@ -868,6 +903,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -940,6 +976,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -972,6 +1010,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1036,12 +1075,17 @@ golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200305205014-bc073721adb6/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo+ZZEYoYFYYDYdOrU7AaM= golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 h1:2ntEwh02rqo2jSsrYmp4yKHHjh0CbXP3ZtSUetSB+q8= +golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1088,12 +1132,21 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -1140,6 +1193,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -1150,6 +1205,8 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc= +honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0= k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= k8s.io/client-go v0.17.3/go.mod h1:cLXlTMtWHkuK4tD360KpWz2gG2KtdWEr/OT02i3emRQ= diff --git a/scripts/install-gometalinter.sh b/scripts/install-gometalinter.sh deleted file mode 100755 index 07612af0a6..0000000000 --- a/scripts/install-gometalinter.sh +++ /dev/null @@ -1,391 +0,0 @@ -#!/bin/sh -# -# Taken from: https://github.com/alecthomas/gometalinter/blob/master/scripts/install.sh - -set -e -# Code generated by godownloader on 2018-05-08T08:53:30Z. DO NOT EDIT. -# - -usage() { - this=$1 - cat </dev/null -} -echoerr() { - echo "$@" 1>&2 -} -log_prefix() { - echo "$0" -} -_logp=6 -log_set_priority() { - _logp="$1" -} -log_priority() { - if test -z "$1"; then - echo "$_logp" - return - fi - [ "$1" -le "$_logp" ] -} -log_tag() { - case $1 in - 0) echo "emerg" ;; - 1) echo "alert" ;; - 2) echo "crit" ;; - 3) echo "err" ;; - 4) echo "warning" ;; - 5) echo "notice" ;; - 6) echo "info" ;; - 7) echo "debug" ;; - *) echo "$1" ;; - esac -} -log_debug() { - log_priority 7 || return 0 - echoerr "$(log_prefix)" "$(log_tag 7)" "$@" -} -log_info() { - log_priority 6 || return 0 - echoerr "$(log_prefix)" "$(log_tag 6)" "$@" -} -log_err() { - log_priority 3 || return 0 - echoerr "$(log_prefix)" "$(log_tag 3)" "$@" -} -log_crit() { - log_priority 2 || return 0 - echoerr "$(log_prefix)" "$(log_tag 2)" "$@" -} -uname_os() { - os=$(uname -s | tr '[:upper:]' '[:lower:]') - case "$os" in - msys_nt) os="windows" ;; - esac - echo "$os" -} -uname_arch() { - arch=$(uname -m) - case $arch in - x86_64) arch="amd64" ;; - x86) arch="386" ;; - i686) arch="386" ;; - i386) arch="386" ;; - aarch64) arch="arm64" ;; - armv5*) arch="armv5" ;; - armv6*) arch="armv6" ;; - armv7*) arch="armv7" ;; - esac - echo ${arch} -} -uname_os_check() { - os=$(uname_os) - case "$os" in - darwin) return 0 ;; - dragonfly) return 0 ;; - freebsd) return 0 ;; - linux) return 0 ;; - android) return 0 ;; - nacl) return 0 ;; - netbsd) return 0 ;; - openbsd) return 0 ;; - plan9) return 0 ;; - solaris) return 0 ;; - windows) return 0 ;; - esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" - return 1 -} -uname_arch_check() { - arch=$(uname_arch) - case "$arch" in - 386) return 0 ;; - amd64) return 0 ;; - arm64) return 0 ;; - armv5) return 0 ;; - armv6) return 0 ;; - armv7) return 0 ;; - ppc64) return 0 ;; - ppc64le) return 0 ;; - mips) return 0 ;; - mipsle) return 0 ;; - mips64) return 0 ;; - mips64le) return 0 ;; - s390x) return 0 ;; - amd64p32) return 0 ;; - esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" - return 1 -} -untar() { - tarball=$1 - case "${tarball}" in - *.tar.gz | *.tgz) tar -xzf "${tarball}" ;; - *.tar) tar -xf "${tarball}" ;; - *.zip) unzip "${tarball}" ;; - *) - log_err "untar unknown archive format for ${tarball}" - return 1 - ;; - esac -} -mktmpdir() { - test -z "$TMPDIR" && TMPDIR="$(mktemp -d)" - mkdir -p "${TMPDIR}" - echo "${TMPDIR}" -} -http_download_curl() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") - else - code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") - fi - if [ "$code" != "200" ]; then - log_debug "http_download_curl received HTTP status $code" - return 1 - fi - return 0 -} -http_download_wget() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - wget -q -O "$local_file" "$source_url" - else - wget -q --header "$header" -O "$local_file" "$source_url" - fi -} -http_download() { - log_debug "http_download $2" - if is_command curl; then - http_download_curl "$@" - return - elif is_command wget; then - http_download_wget "$@" - return - fi - log_crit "http_download unable to find wget or curl" - return 1 -} -http_copy() { - tmp=$(mktemp) - http_download "${tmp}" "$1" "$2" || return 1 - body=$(cat "$tmp") - rm -f "${tmp}" - echo "$body" -} -github_release() { - owner_repo=$1 - version=$2 - test -z "$version" && version="latest" - giturl="https://github.com/${owner_repo}/releases/${version}" - json=$(http_copy "$giturl" "Accept:application/json") - test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') - test -z "$version" && return 1 - echo "$version" -} -hash_sha256() { - TARGET=${1:-/dev/stdin} - if is_command gsha256sum; then - hash=$(gsha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command sha256sum; then - hash=$(sha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command shasum; then - hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command openssl; then - hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f a - else - log_crit "hash_sha256 unable to find command to compute sha-256 hash" - return 1 - fi -} -hash_sha256_verify() { - TARGET=$1 - checksums=$2 - if [ -z "$checksums" ]; then - log_err "hash_sha256_verify checksum file not specified in arg2" - return 1 - fi - BASENAME=${TARGET##*/} - want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) - if [ -z "$want" ]; then - log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" - return 1 - fi - got=$(hash_sha256 "$TARGET") - if [ "$want" != "$got" ]; then - log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" - return 1 - fi -} -cat /dev/null < Date: Thu, 5 Nov 2020 18:32:05 +0200 Subject: [PATCH 26/47] [linter] New value for new-from-rev (#2837) --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 57c383e62c..2d89b4ac65 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -281,4 +281,4 @@ issues: max-same-issues: 0 # Show only new issues created after git revision `REV` - new-from-rev: 57da65a558cc40f7de2f7f5c7528ca6c6278488e + new-from-rev: 5a3a12bba59e058b7a4cee6ec472c3f5786f741b From a3dff4325159919c061c5b0d0b72c99b671159dd Mon Sep 17 00:00:00 2001 From: Gediminas Guoba Date: Fri, 6 Nov 2020 11:18:24 +0200 Subject: [PATCH 27/47] [dbnode] Skip flaky test (#2847) --- src/dbnode/integration/large_tiles_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dbnode/integration/large_tiles_test.go b/src/dbnode/integration/large_tiles_test.go index b8a32739ec..b77b98c2a8 100644 --- a/src/dbnode/integration/large_tiles_test.go +++ b/src/dbnode/integration/large_tiles_test.go @@ -51,6 +51,7 @@ var ( ) func TestReadAggregateWrite(t *testing.T) { + t.Skip("flaky") var ( start = time.Now() testSetup, srcNs, trgNs = setupServer(t) From 4c7eff158c1e3cf9fca03df1efba177b07ba8698 Mon Sep 17 00:00:00 2001 From: Wesley Kim Date: Fri, 6 Nov 2020 10:59:42 -0500 Subject: [PATCH 28/47] Move dice package into dbnode/storage (#2842) --- .golangci.yml | 2 +- src/{x/dice => dbnode/storage}/dice.go | 12 ++++++------ src/{x/dice => dbnode/storage}/dice_test.go | 8 ++++---- src/dbnode/storage/forward_index_dice.go | 6 ++---- src/dbnode/storage/repair.go | 3 +-- 5 files changed, 14 insertions(+), 17 deletions(-) rename src/{x/dice => dbnode/storage}/dice.go (85%) rename src/{x/dice => dbnode/storage}/dice_test.go (94%) diff --git a/.golangci.yml b/.golangci.yml index 2d89b4ac65..df5177f647 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -188,7 +188,6 @@ linters: - typecheck - unconvert - unparam - - unused - varcheck - wsl enable-all: false @@ -209,6 +208,7 @@ linters: - nolintlint # Deprecated project due to being prone to bad suggestions. - interfacer + - unused disable-all: false presets: # bodyclose, errcheck, gosec, govet, scopelint, staticcheck, typecheck diff --git a/src/x/dice/dice.go b/src/dbnode/storage/dice.go similarity index 85% rename from src/x/dice/dice.go rename to src/dbnode/storage/dice.go index fde1a26385..710ee07e6c 100644 --- a/src/x/dice/dice.go +++ b/src/dbnode/storage/dice.go @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package dice +package storage import ( "fmt" @@ -26,17 +26,17 @@ import ( "github.com/MichaelTJones/pcg" ) -// Dice is an interface that allows for random sampling. -type Dice interface { - // Rate returns the sampling rate of this Dice: a number in (0.0, 1.0]. +// dice is an interface that allows for random sampling. +type dice interface { + // Rate returns the sampling rate of this dice: a number in (0.0, 1.0]. Rate() float64 // Roll returns whether the dice roll succeeded. Roll() bool } -// NewDice constructs a new Dice based on a given success rate. -func NewDice(rate float64) (Dice, error) { +// newDice constructs a new dice based on a given success rate. +func newDice(rate float64) (dice, error) { if rate <= 0.0 || rate > 1.0 { return nil, fmt.Errorf("invalid sample rate %f", rate) } diff --git a/src/x/dice/dice_test.go b/src/dbnode/storage/dice_test.go similarity index 94% rename from src/x/dice/dice_test.go rename to src/dbnode/storage/dice_test.go index 523c2b7410..ca018cc29b 100644 --- a/src/x/dice/dice_test.go +++ b/src/dbnode/storage/dice_test.go @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package dice +package storage import ( "testing" @@ -28,17 +28,17 @@ import ( ) func TestDiceConstructor(t *testing.T) { - dice, err := NewDice(0) + dice, err := newDice(0) require.Error(t, err) require.Nil(t, dice) - dice, err = NewDice(2) + dice, err = newDice(2) require.Error(t, err) require.Nil(t, dice) } func TestDice(t *testing.T) { - r, err := NewDice(1) + r, err := newDice(1) require.NoError(t, err) assert.Equal(t, float64(1.0), r.Rate()) diff --git a/src/dbnode/storage/forward_index_dice.go b/src/dbnode/storage/forward_index_dice.go index 30ba4061aa..69c4b767fc 100644 --- a/src/dbnode/storage/forward_index_dice.go +++ b/src/dbnode/storage/forward_index_dice.go @@ -23,8 +23,6 @@ package storage import ( "fmt" "time" - - "github.com/m3db/m3/src/x/dice" ) // forwardIndexDice is a die roll that adds a chance for incoming index writes @@ -36,7 +34,7 @@ type forwardIndexDice struct { blockSize time.Duration forwardIndexThreshold time.Duration - forwardIndexDice dice.Dice + forwardIndexDice dice } func newForwardIndexDice( @@ -72,7 +70,7 @@ func newForwardIndexDice( bufferFragment := float64(bufferFuture) * threshold forwardIndexThreshold = blockSize - time.Duration(bufferFragment) - dice, err := dice.NewDice(probability) + dice, err := newDice(probability) if err != nil { return forwardIndexDice{}, fmt.Errorf("cannot create forward write dice: %s", err) diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 8f6eb523b2..285f2a337f 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -40,7 +40,6 @@ import ( "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/x/context" - "github.com/m3db/m3/src/x/dice" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" @@ -768,7 +767,7 @@ func (r shardRepairer) shadowCompare( shard databaseShard, nsCtx namespace.Context, ) error { - dice, err := dice.NewDice(r.rpopts.DebugShadowComparisonsPercentage()) + dice, err := newDice(r.rpopts.DebugShadowComparisonsPercentage()) if err != nil { return fmt.Errorf("err creating shadow comparison dice: %v", err) } From 5eb4c227597281a9081af9ab09ed161a3c47d193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Fri, 6 Nov 2020 22:52:00 +0200 Subject: [PATCH 29/47] [dbnode] Make open block of downsampled namespace readable (#2777) --- src/dbnode/integration/large_tiles_test.go | 24 +++++++++++---------- src/dbnode/storage/fs.go | 4 ++-- src/dbnode/storage/shard.go | 25 ++++++++++++++++------ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/dbnode/integration/large_tiles_test.go b/src/dbnode/integration/large_tiles_test.go index b77b98c2a8..2f1bd79126 100644 --- a/src/dbnode/integration/large_tiles_test.go +++ b/src/dbnode/integration/large_tiles_test.go @@ -42,16 +42,19 @@ import ( "go.uber.org/zap" ) -var ( +const ( blockSize = 2 * time.Hour blockSizeT = 24 * time.Hour + testDataPointsCount = 60 +) +var ( gaugePayload = &annotation.Payload{MetricType: annotation.MetricType_GAUGE} counterPayload = &annotation.Payload{MetricType: annotation.MetricType_COUNTER, HandleValueResets: true} ) func TestReadAggregateWrite(t *testing.T) { - t.Skip("flaky") +// t.Skip("flaky") var ( start = time.Now() testSetup, srcNs, trgNs = setupServer(t) @@ -68,19 +71,18 @@ func TestReadAggregateWrite(t *testing.T) { session, err := testSetup.M3DBClient().DefaultSession() require.NoError(t, err) - nowFn := testSetup.NowFn() + nowFn := storageOpts.ClockOptions().NowFn() // Write test data. - dpTimeStart := nowFn().Truncate(blockSizeT).Add(-blockSizeT) - dpTime := dpTimeStart + dpTimeStart := nowFn().Truncate(blockSizeT) // "aab" ID is stored to the same shard 0 same as "foo", this is important // for a test to store them to the same shard to test data consistency err = session.WriteTagged(srcNs.ID(), ident.StringID("aab"), ident.MustNewTagStringsIterator("__name__", "cpu", "job", "job1"), - dpTime, 15, xtime.Second, annotationBytes(t, gaugePayload)) + dpTimeStart, 15, xtime.Second, annotationBytes(t, gaugePayload)) - testDataPointsCount := 60 + dpTime := dpTimeStart for a := 0; a < testDataPointsCount; a++ { if a < 10 { dpTime = dpTime.Add(10 * time.Minute) @@ -139,7 +141,7 @@ func TestReadAggregateWrite(t *testing.T) { log.Info("waiting till aggregated data is readable") start = time.Now() readable := xclock.WaitUntil(func() bool { - series, err := session.Fetch(trgNs.ID(), ident.StringID("foo"), dpTimeStart, nowFn()) + series, err := session.Fetch(trgNs.ID(), ident.StringID("foo"), dpTimeStart, dpTimeStart.Add(blockSizeT)) require.NoError(t, err) return series.Next() }, time.Minute) @@ -204,8 +206,8 @@ func fetchAndValidate( func setupServer(t *testing.T) (TestSetup, namespace.Metadata, namespace.Metadata) { var ( - rOpts = retention.NewOptions().SetRetentionPeriod(500 * blockSize).SetBlockSize(blockSize) - rOptsT = retention.NewOptions().SetRetentionPeriod(100 * blockSize).SetBlockSize(blockSizeT).SetBufferPast(0) + rOpts = retention.NewOptions().SetRetentionPeriod(500 * blockSize).SetBlockSize(blockSize).SetBufferPast(0).SetBufferFuture(0) + rOptsT = retention.NewOptions().SetRetentionPeriod(100 * blockSize).SetBlockSize(blockSizeT).SetBufferPast(0).SetBufferFuture(0) idxOpts = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSize) idxOptsT = namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(blockSizeT) nsOpts = namespace.NewOptions(). @@ -216,7 +218,7 @@ func setupServer(t *testing.T) (TestSetup, namespace.Metadata, namespace.Metadat SetRetentionOptions(rOptsT). SetIndexOptions(idxOptsT) - fixedNow = time.Now().Truncate(blockSizeT) + fixedNow = time.Now().Truncate(blockSizeT).Add(11*blockSize) ) srcNs, err := namespace.NewMetadata(testNamespaces[0], nsOpts) diff --git a/src/dbnode/storage/fs.go b/src/dbnode/storage/fs.go index 0fd7509982..e6363f75f0 100644 --- a/src/dbnode/storage/fs.go +++ b/src/dbnode/storage/fs.go @@ -60,8 +60,8 @@ type fileOpState struct { // BlockLeaseVerifier needs to know that a higher cold flush version exists on disk so that // it can approve the SeekerManager's request to open a lease on the latest version. // - // In other words ColdVersionRetrievabled is used to keep track of the latest cold version that has - // been succesfully flushed and can be queried via the block retriever / seeker manager and + // In other words ColdVersionRetrievable is used to keep track of the latest cold version that has + // been successfully flushed and can be queried via the block retriever / seeker manager and // as a result is safe to evict, while ColdVersionFlushed is used to keep track of the latest // cold version that has been flushed and to validate lease requests from the SeekerManager when it // receives a signal to open a new lease. diff --git a/src/dbnode/storage/shard.go b/src/dbnode/storage/shard.go index 292a5dfb05..fecf7cb4dc 100644 --- a/src/dbnode/storage/shard.go +++ b/src/dbnode/storage/shard.go @@ -2850,7 +2850,10 @@ func (s *dbShard) AggregateTiles( // Notify all block leasers that a new volume for the namespace/shard/blockstart // has been created. This will block until all leasers have relinquished their // leases. - if err = s.finishWriting(opts.Start, nextVolume); err != nil { + // NB: markWarmFlushStateSuccess=true because there are no flushes happening in this + // flow and we need to set WarmStatus to fileOpSuccess explicitly in order to make + // the new blocks readable. + if err = s.finishWriting(opts.Start, nextVolume, true); err != nil { multiErr = multiErr.Add(err) } } @@ -2999,14 +3002,22 @@ func (s *dbShard) logFlushResult(r dbShardFlushResult) { ) } -func (s *dbShard) finishWriting(startTime time.Time, nextVersion int) error { +func (s *dbShard) finishWriting( + blockStart time.Time, + nextVersion int, + markWarmFlushStateSuccess bool, +) error { + if markWarmFlushStateSuccess { + s.markWarmFlushStateSuccess(blockStart) + } + // After writing the full block successfully update the ColdVersionFlushed number. This will // allow the SeekerManager to open a lease on the latest version of the fileset files because // the BlockLeaseVerifier will check the ColdVersionFlushed value, but the buffer only looks at // ColdVersionRetrievable so a concurrent tick will not yet cause the blocks in memory to be // evicted (which is the desired behavior because we haven't updated the open leases yet which // means the newly written data is not available for querying via the SeekerManager yet.) - s.setFlushStateColdVersionFlushed(startTime, nextVersion) + s.setFlushStateColdVersionFlushed(blockStart, nextVersion) // Notify all block leasers that a new volume for the namespace/shard/blockstart // has been created. This will block until all leasers have relinquished their @@ -3014,7 +3025,7 @@ func (s *dbShard) finishWriting(startTime time.Time, nextVersion int) error { _, err := s.opts.BlockLeaseManager().UpdateOpenLeases(block.LeaseDescriptor{ Namespace: s.namespace.ID(), Shard: s.ID(), - BlockStart: startTime, + BlockStart: blockStart, }, block.LeaseState{Volume: nextVersion}) // After writing the full block successfully **and** propagating the new lease to the // BlockLeaseManager, update the ColdVersionRetrievable in the flush state. Once this function @@ -3026,13 +3037,13 @@ func (s *dbShard) finishWriting(startTime time.Time, nextVersion int) error { // succeeded, but that would allow the ColdVersionRetrievable and ColdVersionFlushed numbers to drift // which would increase the complexity of the code to address a situation that is probably not // recoverable (failure to UpdateOpenLeases is an invariant violated error). - s.setFlushStateColdVersionRetrievable(startTime, nextVersion) + s.setFlushStateColdVersionRetrievable(blockStart, nextVersion) if err != nil { instrument.EmitAndLogInvariantViolation(s.opts.InstrumentOptions(), func(l *zap.Logger) { l.With( zap.String("namespace", s.namespace.ID().String()), zap.Uint32("shard", s.ID()), - zap.Time("blockStart", startTime), + zap.Time("blockStart", blockStart), zap.Int("nextVersion", nextVersion), ).Error("failed to update open leases after updating flush state cold version") }) @@ -3063,7 +3074,7 @@ func (s shardColdFlush) Done() error { continue } - err := s.shard.finishWriting(startTime, nextVersion) + err := s.shard.finishWriting(startTime, nextVersion, false) if err != nil { multiErr = multiErr.Add(err) } From e62d2e152b211ce5daed6ae74da383d179286aa7 Mon Sep 17 00:00:00 2001 From: Ryan Hall Date: Fri, 6 Nov 2020 15:17:18 -0800 Subject: [PATCH 30/47] Remove pre-checkout hook and fix pre-exit (#2851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pre-checkout is not a repository hook so this never runs. pre-exit was adding noise to the build output. just check if the directory exists first. "chmod: cannot access ‘./_tools’: No such file or directory" --- .buildkite/hooks/pre-checkout | 6 ------ .buildkite/hooks/pre-exit | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100755 .buildkite/hooks/pre-checkout diff --git a/.buildkite/hooks/pre-checkout b/.buildkite/hooks/pre-checkout deleted file mode 100755 index c19519b3b6..0000000000 --- a/.buildkite/hooks/pre-checkout +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -echo "--- :git: cleaning checkout" -chmod -R +w ./_tools diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 497ac79f51..db05d33b76 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -3,5 +3,7 @@ set -eo pipefail echo "--- :git: cleaning checkout" -chmod -R +w ./_tools || true +if [[ -d "./_tools" ]]; then + chmod -R +w ./_tools +fi git clean -dffx From 30b81ca639311ced308d8100498447d8408b8cb0 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Fri, 6 Nov 2020 18:40:13 -0500 Subject: [PATCH 31/47] [dbnode] Remove duplicated clock package (conflicting with src/x/clock) (#2838) --- src/cmd/tools/dtest/harness/harness.go | 7 ++- src/dbnode/client/client_mock.go | 2 +- src/dbnode/client/host_queue.go | 2 +- src/dbnode/client/options.go | 2 +- src/dbnode/client/session.go | 2 +- src/dbnode/client/types.go | 2 +- src/dbnode/clock/options.go | 46 ------------------- src/dbnode/clock/types.go | 37 --------------- ...mmitlog_bootstrap_index_perf_speed_test.go | 2 +- src/dbnode/integration/generate/options.go | 2 +- src/dbnode/integration/generate/types.go | 2 +- src/dbnode/integration/setup.go | 2 +- .../server/tchannelthrift/node/service.go | 2 +- .../network/server/tchannelthrift/options.go | 2 +- .../network/server/tchannelthrift/types.go | 2 +- src/dbnode/persist/fs/commitlog/commit_log.go | 2 +- .../persist/fs/commitlog/commit_log_mock.go | 4 +- .../persist/fs/commitlog/commit_log_test.go | 2 +- src/dbnode/persist/fs/commitlog/options.go | 2 +- src/dbnode/persist/fs/commitlog/types.go | 2 +- src/dbnode/persist/fs/commitlog/writer.go | 2 +- src/dbnode/persist/fs/options.go | 2 +- src/dbnode/persist/fs/persist_manager.go | 2 +- src/dbnode/persist/fs/types.go | 2 +- src/dbnode/storage/block/wired_list.go | 2 +- src/dbnode/storage/block/wired_list_test.go | 2 +- src/dbnode/storage/bootstrap.go | 2 +- .../bootstrap/bootstrapper/fs/source.go | 2 +- .../bootstrap/bootstrapper/peers/source.go | 2 +- .../storage/bootstrap/bootstrapper/readers.go | 2 +- src/dbnode/storage/bootstrap/process.go | 2 +- .../storage/bootstrap/result/options.go | 2 +- src/dbnode/storage/bootstrap/result/types.go | 2 +- src/dbnode/storage/cleanup.go | 2 +- src/dbnode/storage/database.go | 2 +- src/dbnode/storage/flush.go | 2 +- src/dbnode/storage/index.go | 2 +- src/dbnode/storage/index/index_mock.go | 2 +- src/dbnode/storage/index/options.go | 2 +- src/dbnode/storage/index/segments.go | 2 +- src/dbnode/storage/index/types.go | 2 +- src/dbnode/storage/index_insert_queue.go | 2 +- .../storage/index_queue_forward_write_test.go | 5 +- src/dbnode/storage/index_queue_test.go | 2 +- src/dbnode/storage/mediator.go | 2 +- src/dbnode/storage/namespace.go | 2 +- src/dbnode/storage/options.go | 2 +- src/dbnode/storage/repair.go | 2 +- src/dbnode/storage/series/buffer.go | 2 +- src/dbnode/storage/series/lookup/entry.go | 2 +- src/dbnode/storage/series/options.go | 2 +- src/dbnode/storage/series/series_test.go | 2 +- src/dbnode/storage/series/types.go | 2 +- .../series_wired_list_interaction_test.go | 2 +- src/dbnode/storage/shard.go | 2 +- src/dbnode/storage/shard_insert_queue.go | 2 +- src/dbnode/storage/shard_ref_count_test.go | 7 ++- src/dbnode/storage/storage_mock.go | 2 +- src/dbnode/storage/tick.go | 2 +- src/dbnode/storage/types.go | 2 +- 60 files changed, 64 insertions(+), 150 deletions(-) delete mode 100644 src/dbnode/clock/options.go delete mode 100644 src/dbnode/clock/types.go diff --git a/src/cmd/tools/dtest/harness/harness.go b/src/cmd/tools/dtest/harness/harness.go index 68f8fbb4f3..2a7953e341 100644 --- a/src/cmd/tools/dtest/harness/harness.go +++ b/src/cmd/tools/dtest/harness/harness.go @@ -40,11 +40,10 @@ import ( "github.com/m3db/m3/src/cmd/tools/dtest/config" "github.com/m3db/m3/src/cmd/tools/dtest/util" "github.com/m3db/m3/src/cmd/tools/dtest/util/seed" - xclock "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/namespace" + "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/x/m3em/convert" m3emnode "github.com/m3db/m3/src/dbnode/x/m3em/node" "github.com/m3db/m3/src/m3em/build" @@ -52,7 +51,7 @@ import ( hb "github.com/m3db/m3/src/m3em/generated/proto/heartbeat" "github.com/m3db/m3/src/m3em/node" xgrpc "github.com/m3db/m3/src/m3em/x/grpc" - m3xclock "github.com/m3db/m3/src/x/clock" + xclock "github.com/m3db/m3/src/x/clock" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" @@ -419,7 +418,7 @@ func (dt *DTestHarness) WaitUntilAllBootstrapped(nodes []node.ServiceNode) error // available, or the configured bootstrap timeout period; whichever is sooner. It returns // an error indicating if all the nodes finished bootstrapping. func (dt *DTestHarness) WaitUntilAllShardsAvailable() error { - allAvailable := m3xclock.WaitUntil(dt.AllShardsAvailable, dt.BootstrapTimeout()) + allAvailable := xclock.WaitUntil(dt.AllShardsAvailable, dt.BootstrapTimeout()) if !allAvailable { return fmt.Errorf("all shards not available") } diff --git a/src/dbnode/client/client_mock.go b/src/dbnode/client/client_mock.go index f56af6000f..1c623c1e23 100644 --- a/src/dbnode/client/client_mock.go +++ b/src/dbnode/client/client_mock.go @@ -28,7 +28,6 @@ import ( "reflect" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/namespace" @@ -37,6 +36,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/topology" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/client/host_queue.go b/src/dbnode/client/host_queue.go index d9095b38db..32a42dbb94 100644 --- a/src/dbnode/client/host_queue.go +++ b/src/dbnode/client/host_queue.go @@ -27,9 +27,9 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/topology" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" xsync "github.com/m3db/m3/src/x/sync" diff --git a/src/dbnode/client/options.go b/src/dbnode/client/options.go index a54ac12d3d..e36001ff83 100644 --- a/src/dbnode/client/options.go +++ b/src/dbnode/client/options.go @@ -27,7 +27,6 @@ import ( "runtime" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/encoding/m3tsz" "github.com/m3db/m3/src/dbnode/encoding/proto" @@ -38,6 +37,7 @@ import ( m3dbruntime "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/topology" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index c6cd514ea5..f803750d0b 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -32,7 +32,6 @@ import ( "time" "github.com/m3db/m3/src/cluster/shard" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/digest" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" @@ -48,6 +47,7 @@ import ( "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/client/types.go b/src/dbnode/client/types.go index 9d5b47926e..57b8ca71a6 100644 --- a/src/dbnode/client/types.go +++ b/src/dbnode/client/types.go @@ -23,7 +23,6 @@ package client import ( "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/namespace" @@ -32,6 +31,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/topology" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/clock/options.go b/src/dbnode/clock/options.go deleted file mode 100644 index fd80364a4e..0000000000 --- a/src/dbnode/clock/options.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package clock - -import ( - "time" -) - -type options struct { - nowFn NowFn -} - -// NewOptions creates new clock options -func NewOptions() Options { - return &options{ - nowFn: time.Now, - } -} - -func (o *options) SetNowFn(value NowFn) Options { - opts := *o - opts.nowFn = value - return &opts -} - -func (o *options) NowFn() NowFn { - return o.nowFn -} diff --git a/src/dbnode/clock/types.go b/src/dbnode/clock/types.go deleted file mode 100644 index 0b3d5a49f8..0000000000 --- a/src/dbnode/clock/types.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package clock - -import ( - "time" -) - -// NowFn is the function supplied to determine "now" -type NowFn func() time.Time - -// Options represents the options for the clock -type Options interface { - // SetNowFn sets the nowFn - SetNowFn(value NowFn) Options - - // NowFn returns the nowFn - NowFn() NowFn -} diff --git a/src/dbnode/integration/commitlog_bootstrap_index_perf_speed_test.go b/src/dbnode/integration/commitlog_bootstrap_index_perf_speed_test.go index 5d056e9750..ecc8e13428 100644 --- a/src/dbnode/integration/commitlog_bootstrap_index_perf_speed_test.go +++ b/src/dbnode/integration/commitlog_bootstrap_index_perf_speed_test.go @@ -33,13 +33,13 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/persist/fs/commitlog" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/integration/generate/options.go b/src/dbnode/integration/generate/options.go index 186f5dc7f6..fec5b7d7de 100644 --- a/src/dbnode/integration/generate/options.go +++ b/src/dbnode/integration/generate/options.go @@ -24,9 +24,9 @@ import ( "os" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/encoding/m3tsz" + "github.com/m3db/m3/src/x/clock" ) const ( diff --git a/src/dbnode/integration/generate/types.go b/src/dbnode/integration/generate/types.go index 118dd867f7..f716fc412e 100644 --- a/src/dbnode/integration/generate/types.go +++ b/src/dbnode/integration/generate/types.go @@ -24,11 +24,11 @@ import ( "os" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" ns "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/sharding" "github.com/m3db/m3/src/dbnode/ts" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" xtime "github.com/m3db/m3/src/x/time" ) diff --git a/src/dbnode/integration/setup.go b/src/dbnode/integration/setup.go index cf02d7c939..b14123c4d3 100644 --- a/src/dbnode/integration/setup.go +++ b/src/dbnode/integration/setup.go @@ -34,7 +34,6 @@ import ( "github.com/m3db/m3/src/cluster/services" "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/integration/fake" "github.com/m3db/m3/src/dbnode/integration/generate" @@ -56,6 +55,7 @@ import ( "github.com/m3db/m3/src/dbnode/testdata/prototest" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/ts" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" xsync "github.com/m3db/m3/src/x/sync" diff --git a/src/dbnode/network/server/tchannelthrift/node/service.go b/src/dbnode/network/server/tchannelthrift/node/service.go index 19a973c317..99d6c9cc51 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service.go +++ b/src/dbnode/network/server/tchannelthrift/node/service.go @@ -29,7 +29,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/network/server/tchannelthrift" "github.com/m3db/m3/src/dbnode/network/server/tchannelthrift/convert" @@ -42,6 +41,7 @@ import ( "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xdebug "github.com/m3db/m3/src/x/debug" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/network/server/tchannelthrift/options.go b/src/dbnode/network/server/tchannelthrift/options.go index 4b59a61de0..6e8581e89d 100644 --- a/src/dbnode/network/server/tchannelthrift/options.go +++ b/src/dbnode/network/server/tchannelthrift/options.go @@ -21,9 +21,9 @@ package tchannelthrift import ( - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/x/xpool" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" diff --git a/src/dbnode/network/server/tchannelthrift/types.go b/src/dbnode/network/server/tchannelthrift/types.go index 092561257b..9bd0ce03aa 100644 --- a/src/dbnode/network/server/tchannelthrift/types.go +++ b/src/dbnode/network/server/tchannelthrift/types.go @@ -21,9 +21,9 @@ package tchannelthrift import ( - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/x/xpool" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/serialize" diff --git a/src/dbnode/persist/fs/commitlog/commit_log.go b/src/dbnode/persist/fs/commitlog/commit_log.go index 40ab3b8298..0281628aa7 100644 --- a/src/dbnode/persist/fs/commitlog/commit_log.go +++ b/src/dbnode/persist/fs/commitlog/commit_log.go @@ -27,11 +27,11 @@ import ( "sync/atomic" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/persist/fs/commitlog/commit_log_mock.go b/src/dbnode/persist/fs/commitlog/commit_log_mock.go index fe0fa8c15c..f811ffa927 100644 --- a/src/dbnode/persist/fs/commitlog/commit_log_mock.go +++ b/src/dbnode/persist/fs/commitlog/commit_log_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/m3db/m3/src/dbnode/persist/fs/commitlog/types.go -// Copyright (c) 2019 Uber Technologies, Inc. +// Copyright (c) 2020 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,11 +28,11 @@ import ( "reflect" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/persist/fs/commitlog/commit_log_test.go b/src/dbnode/persist/fs/commitlog/commit_log_test.go index ea4894aeaa..00f0601a8a 100644 --- a/src/dbnode/persist/fs/commitlog/commit_log_test.go +++ b/src/dbnode/persist/fs/commitlog/commit_log_test.go @@ -34,12 +34,12 @@ import ( "time" "github.com/m3db/bitset" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/persist/fs/commitlog/options.go b/src/dbnode/persist/fs/commitlog/options.go index 7e43231a4d..fee984e9a6 100644 --- a/src/dbnode/persist/fs/commitlog/options.go +++ b/src/dbnode/persist/fs/commitlog/options.go @@ -26,8 +26,8 @@ import ( "runtime" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist/fs" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" diff --git a/src/dbnode/persist/fs/commitlog/types.go b/src/dbnode/persist/fs/commitlog/types.go index 05ff965db9..ee54adea95 100644 --- a/src/dbnode/persist/fs/commitlog/types.go +++ b/src/dbnode/persist/fs/commitlog/types.go @@ -23,11 +23,11 @@ package commitlog import ( "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/persist/fs/commitlog/writer.go b/src/dbnode/persist/fs/commitlog/writer.go index 864184e533..2040cd38d6 100644 --- a/src/dbnode/persist/fs/commitlog/writer.go +++ b/src/dbnode/persist/fs/commitlog/writer.go @@ -28,13 +28,13 @@ import ( "os" "github.com/m3db/bitset" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/digest" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/persist/fs/msgpack" "github.com/m3db/m3/src/dbnode/persist/schema" "github.com/m3db/m3/src/dbnode/ts" + "github.com/m3db/m3/src/x/clock" xos "github.com/m3db/m3/src/x/os" xtime "github.com/m3db/m3/src/x/time" ) diff --git a/src/dbnode/persist/fs/options.go b/src/dbnode/persist/fs/options.go index 6aa6d246ef..3ececbe1a2 100644 --- a/src/dbnode/persist/fs/options.go +++ b/src/dbnode/persist/fs/options.go @@ -25,10 +25,10 @@ import ( "fmt" "os" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist/fs/msgpack" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/m3ninx/index/segment/fst" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/mmap" "github.com/m3db/m3/src/x/pool" diff --git a/src/dbnode/persist/fs/persist_manager.go b/src/dbnode/persist/fs/persist_manager.go index 69a45d2bb2..37e1021b8c 100644 --- a/src/dbnode/persist/fs/persist_manager.go +++ b/src/dbnode/persist/fs/persist_manager.go @@ -26,7 +26,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/ratelimit" "github.com/m3db/m3/src/dbnode/runtime" @@ -35,6 +34,7 @@ import ( m3ninxfs "github.com/m3db/m3/src/m3ninx/index/segment/fst" m3ninxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/persist/fs/types.go b/src/dbnode/persist/fs/types.go index 7aa7d9fee6..d043f9b3ce 100644 --- a/src/dbnode/persist/fs/types.go +++ b/src/dbnode/persist/fs/types.go @@ -25,7 +25,6 @@ import ( "os" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" @@ -42,6 +41,7 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/fst" idxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/block/wired_list.go b/src/dbnode/storage/block/wired_list.go index e804188b88..ba2ff0c551 100644 --- a/src/dbnode/storage/block/wired_list.go +++ b/src/dbnode/storage/block/wired_list.go @@ -55,8 +55,8 @@ import ( "sync/atomic" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/runtime" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" "github.com/uber-go/tally" diff --git a/src/dbnode/storage/block/wired_list_test.go b/src/dbnode/storage/block/wired_list_test.go index 7a473ec0ce..9cc763e8bc 100644 --- a/src/dbnode/storage/block/wired_list_test.go +++ b/src/dbnode/storage/block/wired_list_test.go @@ -26,11 +26,11 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/bootstrap.go b/src/dbnode/storage/bootstrap.go index 2c569aee3a..a154e09493 100644 --- a/src/dbnode/storage/bootstrap.go +++ b/src/dbnode/storage/bootstrap.go @@ -26,8 +26,8 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/bootstrap" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go index e72c05c1cd..90706d74d3 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go @@ -25,7 +25,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -46,6 +45,7 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/fst" idxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go index 0ab602644e..3f09a360ef 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go @@ -28,7 +28,6 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -45,6 +44,7 @@ import ( "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/index/segment/fst" idxpersist "github.com/m3db/m3/src/m3ninx/persist" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/bootstrap/bootstrapper/readers.go b/src/dbnode/storage/bootstrap/bootstrapper/readers.go index bc158aaae7..d9c6cbacc2 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/readers.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/readers.go @@ -24,12 +24,12 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/bootstrap" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" + "github.com/m3db/m3/src/x/clock" xtime "github.com/m3db/m3/src/x/time" "github.com/opentracing/opentracing-go" diff --git a/src/dbnode/storage/bootstrap/process.go b/src/dbnode/storage/bootstrap/process.go index f28df53883..fb81692b74 100644 --- a/src/dbnode/storage/bootstrap/process.go +++ b/src/dbnode/storage/bootstrap/process.go @@ -25,7 +25,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -33,6 +32,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/tracepoint" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/storage/bootstrap/result/options.go b/src/dbnode/storage/bootstrap/result/options.go index 0be030c398..531e972a8a 100644 --- a/src/dbnode/storage/bootstrap/result/options.go +++ b/src/dbnode/storage/bootstrap/result/options.go @@ -21,9 +21,9 @@ package result import ( - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/series" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" ) diff --git a/src/dbnode/storage/bootstrap/result/types.go b/src/dbnode/storage/bootstrap/result/types.go index 423553ee22..f0df61f114 100644 --- a/src/dbnode/storage/bootstrap/result/types.go +++ b/src/dbnode/storage/bootstrap/result/types.go @@ -23,11 +23,11 @@ package result import ( "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/m3ninx/index/segment" "github.com/m3db/m3/src/m3ninx/persist" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/storage/cleanup.go b/src/dbnode/storage/cleanup.go index d91db51329..96dde48304 100644 --- a/src/dbnode/storage/cleanup.go +++ b/src/dbnode/storage/cleanup.go @@ -26,11 +26,11 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/persist/fs/commitlog" "github.com/m3db/m3/src/dbnode/retention" + "github.com/m3db/m3/src/x/clock" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/database.go b/src/dbnode/storage/database.go index 495387fe44..77e7fa4236 100644 --- a/src/dbnode/storage/database.go +++ b/src/dbnode/storage/database.go @@ -28,7 +28,6 @@ import ( "sync/atomic" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist/fs/commitlog" "github.com/m3db/m3/src/dbnode/persist/fs/wide" @@ -41,6 +40,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/flush.go b/src/dbnode/storage/flush.go index 13b04d757b..d081579f32 100644 --- a/src/dbnode/storage/flush.go +++ b/src/dbnode/storage/flush.go @@ -26,10 +26,10 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs/commitlog" "github.com/m3db/m3/src/dbnode/retention" + "github.com/m3db/m3/src/x/clock" xerrors "github.com/m3db/m3/src/x/errors" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/storage/index.go b/src/dbnode/storage/index.go index 1aa73d162f..9dabe60f96 100644 --- a/src/dbnode/storage/index.go +++ b/src/dbnode/storage/index.go @@ -31,7 +31,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -53,6 +52,7 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment" "github.com/m3db/m3/src/m3ninx/index/segment/builder" idxpersist "github.com/m3db/m3/src/m3ninx/persist" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/storage/index/index_mock.go b/src/dbnode/storage/index/index_mock.go index 2aede296a2..d98d3c231b 100644 --- a/src/dbnode/storage/index/index_mock.go +++ b/src/dbnode/storage/index/index_mock.go @@ -28,7 +28,6 @@ import ( "reflect" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index/compaction" "github.com/m3db/m3/src/dbnode/storage/limits" @@ -37,6 +36,7 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/builder" "github.com/m3db/m3/src/m3ninx/index/segment/fst" "github.com/m3db/m3/src/m3ninx/index/segment/mem" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/index/options.go b/src/dbnode/storage/index/options.go index ba5243272e..3ff3e9dd1e 100644 --- a/src/dbnode/storage/index/options.go +++ b/src/dbnode/storage/index/options.go @@ -23,13 +23,13 @@ package index import ( "errors" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/index/compaction" "github.com/m3db/m3/src/dbnode/storage/limits" "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/index/segment/builder" "github.com/m3db/m3/src/m3ninx/index/segment/fst" "github.com/m3db/m3/src/m3ninx/index/segment/mem" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/mmap" diff --git a/src/dbnode/storage/index/segments.go b/src/dbnode/storage/index/segments.go index 837b9fc325..ce3d8ae5b3 100644 --- a/src/dbnode/storage/index/segments.go +++ b/src/dbnode/storage/index/segments.go @@ -23,8 +23,8 @@ package index import ( "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/m3ninx/index/segment" + "github.com/m3db/m3/src/x/clock" ) type readableSeg struct { diff --git a/src/dbnode/storage/index/types.go b/src/dbnode/storage/index/types.go index 88323e7909..58afff9776 100644 --- a/src/dbnode/storage/index/types.go +++ b/src/dbnode/storage/index/types.go @@ -25,7 +25,6 @@ import ( "sort" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index/compaction" @@ -36,6 +35,7 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/builder" "github.com/m3db/m3/src/m3ninx/index/segment/fst" "github.com/m3db/m3/src/m3ninx/index/segment/mem" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/index_insert_queue.go b/src/dbnode/storage/index_insert_queue.go index f3edbbc7c8..b168b99879 100644 --- a/src/dbnode/storage/index_insert_queue.go +++ b/src/dbnode/storage/index_insert_queue.go @@ -26,10 +26,10 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/ts/writes" + "github.com/m3db/m3/src/x/clock" xsync "github.com/m3db/m3/src/x/sync" "github.com/uber-go/tally" diff --git a/src/dbnode/storage/index_queue_forward_write_test.go b/src/dbnode/storage/index_queue_forward_write_test.go index dcc096b882..15804cf2b4 100644 --- a/src/dbnode/storage/index_queue_forward_write_test.go +++ b/src/dbnode/storage/index_queue_forward_write_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/index" @@ -35,7 +34,7 @@ import ( xmetrics "github.com/m3db/m3/src/dbnode/x/metrics" "github.com/m3db/m3/src/m3ninx/doc" m3ninxidx "github.com/m3db/m3/src/m3ninx/idx" - xclock "github.com/m3db/m3/src/x/clock" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" xtest "github.com/m3db/m3/src/x/test" @@ -494,7 +493,7 @@ func verifyShard( next time.Time, id string, ) { - allQueriesSuccess := xclock.WaitUntil(func() bool { + allQueriesSuccess := clock.WaitUntil(func() bool { query := m3ninxidx.NewFieldQuery([]byte(id)) // check current index block for series res, err := idx.Query(ctx, index.Query{Query: query}, index.QueryOptions{ diff --git a/src/dbnode/storage/index_queue_test.go b/src/dbnode/storage/index_queue_test.go index 4fe044bebf..1b1c3e2e4c 100644 --- a/src/dbnode/storage/index_queue_test.go +++ b/src/dbnode/storage/index_queue_test.go @@ -26,12 +26,12 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" m3dberrors "github.com/m3db/m3/src/dbnode/storage/errors" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/m3ninx/doc" m3ninxidx "github.com/m3db/m3/src/m3ninx/idx" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" xtest "github.com/m3db/m3/src/x/test" diff --git a/src/dbnode/storage/mediator.go b/src/dbnode/storage/mediator.go index b0afb18896..efe70159e7 100644 --- a/src/dbnode/storage/mediator.go +++ b/src/dbnode/storage/mediator.go @@ -25,9 +25,9 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/dbnode/persist/fs/commitlog" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" "github.com/uber-go/tally" diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index 6465df365d..e535169c8c 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -28,7 +28,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -43,6 +42,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/storage/options.go b/src/dbnode/storage/options.go index 3beba1a5a4..43dbca3de9 100644 --- a/src/dbnode/storage/options.go +++ b/src/dbnode/storage/options.go @@ -29,7 +29,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/encoding/m3tsz" "github.com/m3db/m3/src/dbnode/namespace" @@ -46,6 +45,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 285f2a337f..8c727a6990 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -31,7 +31,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/block" @@ -39,6 +38,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/repair" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/series/buffer.go b/src/dbnode/storage/series/buffer.go index 1e7ed84451..165cf53bb8 100644 --- a/src/dbnode/storage/series/buffer.go +++ b/src/dbnode/storage/series/buffer.go @@ -27,13 +27,13 @@ import ( "sync/atomic" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/series/lookup/entry.go b/src/dbnode/storage/series/lookup/entry.go index 8e1917524e..a1319af9c7 100644 --- a/src/dbnode/storage/series/lookup/entry.go +++ b/src/dbnode/storage/series/lookup/entry.go @@ -25,12 +25,12 @@ import ( "sync/atomic" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/bootstrap" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/dbnode/ts/writes" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xtime "github.com/m3db/m3/src/x/time" ) diff --git a/src/dbnode/storage/series/options.go b/src/dbnode/storage/series/options.go index 682b432e28..155749c03a 100644 --- a/src/dbnode/storage/series/options.go +++ b/src/dbnode/storage/series/options.go @@ -21,11 +21,11 @@ package series import ( - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/retention" m3dbruntime "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/block" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/series/series_test.go b/src/dbnode/storage/series/series_test.go index 7d9d918f5a..3d9441e923 100644 --- a/src/dbnode/storage/series/series_test.go +++ b/src/dbnode/storage/series/series_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/encoding/m3tsz" "github.com/m3db/m3/src/dbnode/persist" @@ -37,6 +36,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/index/convert" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" diff --git a/src/dbnode/storage/series/types.go b/src/dbnode/storage/series/types.go index cf434c8e30..439393e8c4 100644 --- a/src/dbnode/storage/series/types.go +++ b/src/dbnode/storage/series/types.go @@ -23,7 +23,6 @@ package series import ( "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" @@ -33,6 +32,7 @@ import ( "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/series_wired_list_interaction_test.go b/src/dbnode/storage/series_wired_list_interaction_test.go index b446bfa5aa..4a67f9a657 100644 --- a/src/dbnode/storage/series_wired_list_interaction_test.go +++ b/src/dbnode/storage/series_wired_list_interaction_test.go @@ -25,13 +25,13 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/dbnode/storage/series/lookup" "github.com/m3db/m3/src/dbnode/ts" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/shard.go b/src/dbnode/storage/shard.go index fecf7cb4dc..056afa8166 100644 --- a/src/dbnode/storage/shard.go +++ b/src/dbnode/storage/shard.go @@ -29,7 +29,6 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/encoding/tile" "github.com/m3db/m3/src/dbnode/generated/proto/annotation" @@ -54,6 +53,7 @@ import ( "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/storage/shard_insert_queue.go b/src/dbnode/storage/shard_insert_queue.go index fe66bfd45a..794b9848a3 100644 --- a/src/dbnode/storage/shard_insert_queue.go +++ b/src/dbnode/storage/shard_insert_queue.go @@ -26,13 +26,13 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/dbnode/storage/series/lookup" "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/ident" xsync "github.com/m3db/m3/src/x/sync" xtime "github.com/m3db/m3/src/x/time" diff --git a/src/dbnode/storage/shard_ref_count_test.go b/src/dbnode/storage/shard_ref_count_test.go index 1ca60a44ae..517848dc81 100644 --- a/src/dbnode/storage/shard_ref_count_test.go +++ b/src/dbnode/storage/shard_ref_count_test.go @@ -27,13 +27,12 @@ import ( "github.com/fortytw2/leaktest" "github.com/golang/mock/gomock" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/storage/series" xmetrics "github.com/m3db/m3/src/dbnode/x/metrics" - xclock "github.com/m3db/m3/src/x/clock" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" xtime "github.com/m3db/m3/src/x/time" @@ -270,7 +269,7 @@ func TestShardWriteAsyncRefCount(t *testing.T) { assert.NoError(t, err) assert.True(t, seriesWrite.WasWritten) - inserted := xclock.WaitUntil(func() bool { + inserted := clock.WaitUntil(func() bool { counter, ok := testReporter.Counters()["dbshard.insert-queue.inserts"] return ok && counter == 3 }, 2*time.Second) @@ -436,7 +435,7 @@ func testShardWriteTaggedAsyncRefCount(t *testing.T, idx NamespaceIndex, nowFn f seriesWrite.PendingIndexInsert.Entry.OnIndexSeries.OnIndexSuccess(idx.BlockStartForWriteTime(now)) seriesWrite.PendingIndexInsert.Entry.OnIndexSeries.OnIndexFinalize(idx.BlockStartForWriteTime(now)) - inserted := xclock.WaitUntil(func() bool { + inserted := clock.WaitUntil(func() bool { counter, ok := testReporter.Counters()["dbshard.insert-queue.inserts"] return ok && counter == 3 }, 5*time.Second) diff --git a/src/dbnode/storage/storage_mock.go b/src/dbnode/storage/storage_mock.go index feb8fbef7f..d6a3987454 100644 --- a/src/dbnode/storage/storage_mock.go +++ b/src/dbnode/storage/storage_mock.go @@ -30,7 +30,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" @@ -49,6 +48,7 @@ import ( "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" diff --git a/src/dbnode/storage/tick.go b/src/dbnode/storage/tick.go index 33585e8517..8a95d2f54b 100644 --- a/src/dbnode/storage/tick.go +++ b/src/dbnode/storage/tick.go @@ -25,8 +25,8 @@ import ( "sync" "time" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/runtime" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" diff --git a/src/dbnode/storage/types.go b/src/dbnode/storage/types.go index b7753bc9f8..eae5765473 100644 --- a/src/dbnode/storage/types.go +++ b/src/dbnode/storage/types.go @@ -26,7 +26,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/clock" "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/persist" @@ -47,6 +46,7 @@ import ( "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" From c7392e11b3831e60e886bb44aa113c514f150126 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Sat, 7 Nov 2020 00:48:38 -0500 Subject: [PATCH 32/47] [dbnode] [coordinator] Refactor config to use defaults wherever possible (#2839) --- .golangci.yml | 7 +- config/m3db/clustered-etcd/generated.yaml | 1 - config/m3db/clustered-etcd/m3dbnode.libsonnet | 1 - config/m3db/local-etcd/generated.yaml | 1 - config/m3db/local-etcd/m3dbnode.libsonnet | 1 - kube/bundle.yaml | 1 - kube/m3dbnode-configmap.yaml | 1 - kube/terraform/main.tf | 2 +- scripts/development/m3_stack/m3dbnode.yml | 42 ---- .../m3dbnode.yml | 1 - .../m3dbnode-cluster-a.yml | 1 - .../m3dbnode-cluster-b.yml | 1 - .../repair/m3dbnode.yml | 1 - .../m3dbnode-cluster-a.yml | 1 - .../m3dbnode-cluster-b.yml | 1 - .../replication/m3dbnode-cluster-a.yml | 1 - .../replication/m3dbnode-cluster-b.yml | 1 - .../availability_consistency_durability.md | 12 +- src/cmd/services/m3dbnode/config/config.go | 192 +++++++++++++++--- .../services/m3dbnode/config/config_test.go | 9 +- src/cmd/services/m3dbnode/config/limits.go | 3 + src/cmd/services/m3dbnode/main/main.go | 2 +- .../services/m3dbnode/main/main_index_test.go | 1 - src/cmd/services/m3dbnode/main/main_test.go | 1 - src/cmd/services/m3query/config/config.go | 57 +++++- .../harness/resources/config/m3dbnode.yml | 1 - src/dbnode/config/m3dbnode-all-config.yml | 2 - .../config/m3dbnode-cluster-template.yml | 1 - .../config/m3dbnode-local-etcd-proto.yml | 1 - src/dbnode/config/m3dbnode-local-etcd.yml | 61 ------ src/dbnode/runtime/runtime_options.go | 2 +- src/dbnode/server/server.go | 89 ++++---- src/query/api/v1/handler/database/create.go | 2 +- .../api/v1/handler/database/create_test.go | 5 +- src/query/server/multi_process.go | 17 +- src/query/server/query.go | 13 +- 36 files changed, 314 insertions(+), 222 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index df5177f647..5cb97c4887 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -189,7 +189,6 @@ linters: - unconvert - unparam - varcheck - - wsl enable-all: false disable: - gomnd @@ -208,7 +207,11 @@ linters: - nolintlint # Deprecated project due to being prone to bad suggestions. - interfacer - - unused + # Valid use for not explicitly setting every field when they are optional nil/empty. + - exhaustivestruct + # We allow cuddling assignment following conditions because there are valid + # logical groupings for this use-case (e.g. when evaluating config values). + - wsl disable-all: false presets: # bodyclose, errcheck, gosec, govet, scopelint, staticcheck, typecheck diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml index f62e7ada30..435fc03f50 100644 --- a/config/m3db/clustered-etcd/generated.yaml +++ b/config/m3db/clustered-etcd/generated.yaml @@ -73,4 +73,3 @@ "sanitization": "prometheus" "writeNewSeriesAsync": true "writeNewSeriesBackoffDuration": "2ms" - "writeNewSeriesLimitPerSecond": 1048576 diff --git a/config/m3db/clustered-etcd/m3dbnode.libsonnet b/config/m3db/clustered-etcd/m3dbnode.libsonnet index 50bb6fa1ed..39de162dd5 100644 --- a/config/m3db/clustered-etcd/m3dbnode.libsonnet +++ b/config/m3db/clustered-etcd/m3dbnode.libsonnet @@ -101,7 +101,6 @@ function(cluster, coordinator={}, db={}) { }, "gcPercentage": 100, "writeNewSeriesAsync": true, - "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "cache": { "series": { diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index cc5b10c10c..d307c410a9 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -68,4 +68,3 @@ "sanitization": "prometheus" "writeNewSeriesAsync": true "writeNewSeriesBackoffDuration": "2ms" - "writeNewSeriesLimitPerSecond": 1048576 diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet index fc6e5c5651..575733ba98 100644 --- a/config/m3db/local-etcd/m3dbnode.libsonnet +++ b/config/m3db/local-etcd/m3dbnode.libsonnet @@ -60,7 +60,6 @@ function(coordinator={}, db={}) { }, "gcPercentage": 100, "writeNewSeriesAsync": true, - "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "cache": { "series": { diff --git a/kube/bundle.yaml b/kube/bundle.yaml index 8e99d0b72e..8449fcf925 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -163,7 +163,6 @@ data: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms commitlog: diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index f51032e5c9..6ccd3084e3 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -53,7 +53,6 @@ data: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms commitlog: diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 675805edda..6391658abb 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesBackoffDuration: 2ms\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 4243c971c1..35a384a193 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -1,7 +1,4 @@ db: - logging: - level: info - tracing: backend: jaeger jaeger: @@ -11,49 +8,10 @@ db: type: const param: 1 - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - config: service: env: default_env diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index d94b0d3993..23e7bdf9eb 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -45,7 +45,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 5527f12830..124973365a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -56,7 +56,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index edbfcc298b..e1c13baba6 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index b6f5cb5436..eb307f1e81 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index 93dedab49d..f2403c8890 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index b2c5b2c650..bcdd21cb21 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index 9ba35ef331..e89976d488 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index c3097f8df8..195497e169 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -35,7 +35,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/site/content/docs/operational_guide/availability_consistency_durability.md b/site/content/docs/operational_guide/availability_consistency_durability.md index 873ed4c6aa..1f14ac16ee 100644 --- a/site/content/docs/operational_guide/availability_consistency_durability.md +++ b/site/content/docs/operational_guide/availability_consistency_durability.md @@ -57,10 +57,12 @@ This instructs M3DB to handle writes for new timeseries (for a given time block) However, since new time series are created asynchronously, it's possible that there may be a brief delay inbetween when a write is acknowledged by the client and when that series becomes available for subsequent reads. -M3DB also allows operators to rate limit the number of new series that can be created per second via the following configuration: +M3DB also allows operators to rate limit the number of new series that can be created per second via the following configuration under the `db.limits` section: ```yaml -writeNewSeriesLimitPerSecond: 1048576 +db: + limits: + writeNewSeriesPerSecond: 1048576 ``` This value can be set much lower than the default value for workloads in which a significant increase in cardinality usually indicates a misbehaving caller. @@ -105,10 +107,12 @@ writeNewSeriesAsync: false This instructs M3DB to handle writes for new timeseries (for a given time block) synchronously. Creating a new timeseries in memory is much more expensive than simply appending a new write to an existing series, so this configuration could have an adverse effect on performance when many new timeseries are being inserted into M3DB concurrently. -Since this operation is so expensive, M3DB allows operator to rate limit the number of new series that can be created per second via the following configuration (also a top-level key under the `db` section): +Since this operation is so expensive, M3DB allows operator to rate limit the number of new series that can be created per second via the following configuration (also a top-level key under the `db.limits` section): ```yaml -writeNewSeriesLimitPerSecond: 1048576 +db: + limits: + writeNewSeriesPerSecond: 1048576 ``` ### Ignoring Corrupt Commitlogs on Bootstrap diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index e0ebed20da..9f438cec6b 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -51,6 +51,38 @@ const ( defaultEtcdServerPort = 2380 ) +var ( + defaultLogging = xlog.Configuration{ + Level: "info", + } + defaultMetricsSanitization = instrument.PrometheusMetricSanitization + defaultMetricsExtendedMetricsType = instrument.DetailedExtendedMetrics + defaultMetrics = instrument.MetricsConfiguration{ + PrometheusReporter: &instrument.PrometheusConfiguration{ + HandlerPath: "/metrics", + }, + Sanitization: &defaultMetricsSanitization, + SamplingRate: 1.0, + ExtendedMetrics: &defaultMetricsExtendedMetricsType, + } + defaultListenAddress = "0.0.0.0:9000" + defaultClusterListenAddress = "0.0.0.0:9001" + defaultHTTPNodeListenAddress = "0.0.0.0:9002" + defaultHTTPClusterListenAddress = "0.0.0.0:9003" + defaultDebugListenAddress = "0.0.0.0:9004" + defaultGCPercentage = 100 + defaultWriteNewSeriesAsync = true + defaultWriteNewSeriesBackoffDuration = 2 * time.Millisecond + defaultCommitLogPolicy = CommitLogPolicy{ + FlushMaxBytes: 524288, + FlushEvery: time.Second * 1, + Queue: CommitLogQueuePolicy{ + Size: 2097152, + CalculationType: CalculationTypeFixed, + }, + } +) + // Configuration is the top level configuration that includes both a DB // node and a coordinator. type Configuration struct { @@ -61,10 +93,10 @@ type Configuration struct { Coordinator *coordinatorcfg.Configuration `yaml:"coordinator"` } -// InitDefaultsAndValidate initializes all default values and validates the Configuration. -// We use this method to validate fields where the validator package falls short. -func (c *Configuration) InitDefaultsAndValidate() error { - return c.DB.InitDefaultsAndValidate() +// Validate validates the Configuration. We use this method to validate fields +// where the validator package falls short. +func (c *Configuration) Validate() error { + return c.DB.Validate() } // DBConfiguration is the configuration for a DB node. @@ -76,25 +108,25 @@ type DBConfiguration struct { Transforms TransformConfiguration `yaml:"transforms"` // Logging configuration. - Logging xlog.Configuration `yaml:"logging"` + Logging *xlog.Configuration `yaml:"logging"` // Metrics configuration. - Metrics instrument.MetricsConfiguration `yaml:"metrics"` + Metrics *instrument.MetricsConfiguration `yaml:"metrics"` // The host and port on which to listen for the node service. - ListenAddress string `yaml:"listenAddress" validate:"nonzero"` + ListenAddress *string `yaml:"listenAddress"` // The host and port on which to listen for the cluster service. - ClusterListenAddress string `yaml:"clusterListenAddress" validate:"nonzero"` + ClusterListenAddress *string `yaml:"clusterListenAddress"` // The HTTP host and port on which to listen for the node service. - HTTPNodeListenAddress string `yaml:"httpNodeListenAddress" validate:"nonzero"` + HTTPNodeListenAddress *string `yaml:"httpNodeListenAddress"` // The HTTP host and port on which to listen for the cluster service. - HTTPClusterListenAddress string `yaml:"httpClusterListenAddress" validate:"nonzero"` + HTTPClusterListenAddress *string `yaml:"httpClusterListenAddress"` // The host and port on which to listen for debug endpoints. - DebugListenAddress string `yaml:"debugListenAddress"` + DebugListenAddress *string `yaml:"debugListenAddress"` // HostID is the local host ID configuration. HostID hostid.Configuration `yaml:"hostID"` @@ -105,14 +137,6 @@ type DBConfiguration struct { // The initial garbage collection target percentage. GCPercentage int `yaml:"gcPercentage" validate:"max=100"` - // TODO(V1): Move to `limits`. - // Write new series limit per second to limit overwhelming during new ID bursts. - WriteNewSeriesLimitPerSecond int `yaml:"writeNewSeriesLimitPerSecond"` - - // TODO(V1): Move to `limits`. - // Write new series backoff between batches of new series insertions. - WriteNewSeriesBackoffDuration time.Duration `yaml:"writeNewSeriesBackoffDuration"` - // The tick configuration, omit this to use default settings. Tick *TickConfiguration `yaml:"tick"` @@ -129,7 +153,7 @@ type DBConfiguration struct { Filesystem FilesystemConfiguration `yaml:"filesystem"` // The commit log policy for the node. - CommitLog CommitLogPolicy `yaml:"commitlog"` + CommitLog *CommitLogPolicy `yaml:"commitlog"` // The repair policy for repairing data within a cluster. Repair *RepairPolicy `yaml:"repair"` @@ -138,7 +162,7 @@ type DBConfiguration struct { Replication *ReplicationPolicy `yaml:"replication"` // The pooling policy. - PoolingPolicy PoolingPolicy `yaml:"pooling"` + PoolingPolicy *PoolingPolicy `yaml:"pooling"` // The environment (static or dynamic) configuration. EnvironmentConfig environment.Configuration `yaml:"config"` @@ -147,7 +171,10 @@ type DBConfiguration struct { Hashing HashingConfiguration `yaml:"hashing"` // Write new series asynchronously for fast ingestion of new ID bursts. - WriteNewSeriesAsync bool `yaml:"writeNewSeriesAsync"` + WriteNewSeriesAsync *bool `yaml:"writeNewSeriesAsync"` + + // Write new series backoff between batches of new series insertions. + WriteNewSeriesBackoffDuration *time.Duration `yaml:"writeNewSeriesBackoffDuration"` // Proto contains the configuration specific to running in the ProtoDataMode. Proto *ProtoConfiguration `yaml:"proto"` @@ -171,14 +198,127 @@ type DBConfiguration struct { Debug config.DebugConfiguration `yaml:"debug"` } -// InitDefaultsAndValidate initializes all default values and validates the Configuration. -// We use this method to validate fields where the validator package falls short. -func (c *DBConfiguration) InitDefaultsAndValidate() error { +// LoggingOrDefault returns the logging configuration or defaults. +func (c *DBConfiguration) LoggingOrDefault() xlog.Configuration { + if c.Logging == nil { + return defaultLogging + } + + return *c.Logging +} + +// MetricsOrDefault returns metrics configuration or defaults. +func (c *DBConfiguration) MetricsOrDefault() *instrument.MetricsConfiguration { + if c.Metrics == nil { + return &defaultMetrics + } + + return c.Metrics +} + +// ListenAddressOrDefault returns the listen address or default. +func (c *DBConfiguration) ListenAddressOrDefault() string { + if c.ListenAddress == nil { + return defaultListenAddress + } + + return *c.ListenAddress +} + +// ClusterListenAddressOrDefault returns the listen address or default. +func (c *DBConfiguration) ClusterListenAddressOrDefault() string { + if c.ClusterListenAddress == nil { + return defaultClusterListenAddress + } + + return *c.ClusterListenAddress +} + +// HTTPNodeListenAddressOrDefault returns the listen address or default. +func (c *DBConfiguration) HTTPNodeListenAddressOrDefault() string { + if c.HTTPNodeListenAddress == nil { + return defaultHTTPNodeListenAddress + } + + return *c.HTTPNodeListenAddress +} + +// HTTPClusterListenAddressOrDefault returns the listen address or default. +func (c *DBConfiguration) HTTPClusterListenAddressOrDefault() string { + if c.HTTPClusterListenAddress == nil { + return defaultHTTPClusterListenAddress + } + + return *c.HTTPClusterListenAddress +} + +// DebugListenAddressOrDefault returns the listen address or default. +func (c *DBConfiguration) DebugListenAddressOrDefault() string { + if c.DebugListenAddress == nil { + return defaultDebugListenAddress + } + + return *c.DebugListenAddress +} + +// CommitLogOrDefault returns the commit log policy or default. +func (c *DBConfiguration) CommitLogOrDefault() CommitLogPolicy { + if c.CommitLog == nil { + return defaultCommitLogPolicy + } + + return *c.CommitLog +} + +// GCPercentageOrDefault returns the GC percentage or default. +func (c *DBConfiguration) GCPercentageOrDefault() int { + if c.GCPercentage == 0 { + return defaultGCPercentage + } + + return c.GCPercentage +} + +// WriteNewSeriesAsyncOrDefault returns whether to write new series async or not. +func (c *DBConfiguration) WriteNewSeriesAsyncOrDefault() bool { + if c.WriteNewSeriesAsync == nil { + return defaultWriteNewSeriesAsync + } + + return *c.WriteNewSeriesAsync +} + +// WriteNewSeriesBackoffDurationOrDefault returns the backoff duration for new series inserts. +func (c *DBConfiguration) WriteNewSeriesBackoffDurationOrDefault() time.Duration { + if c.WriteNewSeriesBackoffDuration == nil { + return defaultWriteNewSeriesBackoffDuration + } + + return *c.WriteNewSeriesBackoffDuration +} + +// PoolingPolicyOrDefault returns the pooling policy or default. +func (c *DBConfiguration) PoolingPolicyOrDefault() (PoolingPolicy, error) { + var policy PoolingPolicy + if c.PoolingPolicy != nil { + policy = *c.PoolingPolicy + } + + if err := policy.InitDefaultsAndValidate(); err != nil { + return PoolingPolicy{}, err + } + + return policy, nil +} + +// Validate validates the Configuration. We use this method to validate fields +// where the validator package falls short. +func (c *DBConfiguration) Validate() error { if err := c.Filesystem.Validate(); err != nil { return err } - if err := c.PoolingPolicy.InitDefaultsAndValidate(); err != nil { + if _, err := c.PoolingPolicyOrDefault(); err != nil { return err } diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 10c82b710b..2db737a5df 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -95,9 +95,6 @@ db: gcPercentage: 100 - writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms - bootstrap: filesystem: numProcessorsPerCPU: 0.42 @@ -311,7 +308,7 @@ db: hashing: seed: 42 writeNewSeriesAsync: true - + writeNewSeriesBackoffDuration: 2ms tracing: backend: jaeger ` @@ -409,8 +406,6 @@ func TestConfiguration(t *testing.T) { writeShardsInitializing: null shardsLeavingCountTowardsConsistency: null gcPercentage: 100 - writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms tick: null bootstrap: mode: null @@ -669,6 +664,7 @@ func TestConfiguration(t *testing.T) { hashing: seed: 42 writeNewSeriesAsync: true + writeNewSeriesBackoffDuration: 2ms proto: null tracing: serviceName: "" @@ -719,6 +715,7 @@ func TestConfiguration(t *testing.T) { maxOutstandingReadRequests: 0 maxOutstandingRepairedBytes: 0 maxEncodersPerBlock: 0 + writeNewSeriesPerSecond: 0 wide: null tchannel: null debug: diff --git a/src/cmd/services/m3dbnode/config/limits.go b/src/cmd/services/m3dbnode/config/limits.go index 039d08f690..29c6dafe78 100644 --- a/src/cmd/services/m3dbnode/config/limits.go +++ b/src/cmd/services/m3dbnode/config/limits.go @@ -58,6 +58,9 @@ type LimitsConfiguration struct { // load on the CPU, which can prevent other DB operations. // A setting of 0 means there is no maximum. MaxEncodersPerBlock int `yaml:"maxEncodersPerBlock" validate:"min=0"` + + // Write new series limit per second to limit overwhelming during new ID bursts. + WriteNewSeriesPerSecond int `yaml:"writeNewSeriesPerSecond" validate:"min=0"` } // MaxRecentQueryResourceLimitConfiguration sets an upper limit on resources consumed by all queries diff --git a/src/cmd/services/m3dbnode/main/main.go b/src/cmd/services/m3dbnode/main/main.go index 9112a6d8cd..81d006d4b5 100644 --- a/src/cmd/services/m3dbnode/main/main.go +++ b/src/cmd/services/m3dbnode/main/main.go @@ -56,7 +56,7 @@ func main() { os.Exit(1) } - if err := cfg.InitDefaultsAndValidate(); err != nil { + if err := cfg.Validate(); err != nil { // NB(r): Use fmt.Fprintf(os.Stderr, ...) to avoid etcd.SetGlobals() // sending stdlib "log" to black hole. Don't remove unless with good reason. fmt.Fprintf(os.Stderr, "erro validating config: %v\n", err) diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index a4cba46dbf..276c01bfde 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -347,7 +347,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: false - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms commitlog: diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index d8068c6e29..0bb417b2d7 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -503,7 +503,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms commitlog: diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index e1da5d5e13..093f490fc7 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -57,6 +57,8 @@ const ( // coordinators used only to serve m3admin APIs. NoopEtcdStorageType BackendStorageType = "noop-etcd" + defaultListenAddress = "0.0.0.0:7201" + defaultCarbonIngesterListenAddress = "0.0.0.0:7204" defaultQueryTimeout = 30 * time.Second @@ -65,6 +67,25 @@ const ( ) var ( + defaultLogging = xlog.Configuration{ + Level: "info", + } + defaultMetricsSanitization = instrument.PrometheusMetricSanitization + defaultMetricsExtendedMetricsType = instrument.NoExtendedMetrics + defaultMetrics = instrument.MetricsConfiguration{ + RootScope: &instrument.ScopeConfiguration{ + Prefix: "coordinator", + }, + PrometheusReporter: &instrument.PrometheusConfiguration{ + HandlerPath: "/metrics", + // Default to coordinator (until https://github.com/m3db/m3/issues/682 is resolved) + ListenAddress: "0.0.0.0:7203", + }, + Sanitization: &defaultMetricsSanitization, + SamplingRate: 1.0, + ExtendedMetrics: &defaultMetricsExtendedMetricsType, + } + // 5m is the default lookback in Prometheus defaultLookbackDuration = 5 * time.Minute @@ -89,10 +110,10 @@ var ( // Configuration is the configuration for the query service. type Configuration struct { // Metrics configuration. - Metrics instrument.MetricsConfiguration `yaml:"metrics"` + Metrics *instrument.MetricsConfiguration `yaml:"metrics"` // Logging configuration. - Logging xlog.Configuration `yaml:"logging"` + Logging *xlog.Configuration `yaml:"logging"` // Tracing configures opentracing. If not provided, tracing is disabled. Tracing opentracing.TracingConfiguration `yaml:"tracing"` @@ -110,7 +131,7 @@ type Configuration struct { ClusterManagement *ClusterManagementConfiguration `yaml:"clusterManagement"` // ListenAddress is the server listen address. - ListenAddress string `yaml:"listenAddress" validate:"nonzero"` + ListenAddress *string `yaml:"listenAddress"` // Filter is the read/write/complete tags filter configuration. Filter FilterConfiguration `yaml:"filter"` @@ -167,11 +188,39 @@ type Configuration struct { Debug config.DebugConfiguration `yaml:"debug"` } +// ListenAddressOrDefault returns the listen address or default. +func (c *Configuration) ListenAddressOrDefault() string { + if c.ListenAddress != nil { + return *c.ListenAddress + } + + return defaultListenAddress +} + +// LoggingOrDefault returns the logging config or default. +func (c *Configuration) LoggingOrDefault() xlog.Configuration { + if c.Logging != nil { + return *c.Logging + } + + return defaultLogging +} + +// MetricsOrDefault returns the metrics config or default. +func (c *Configuration) MetricsOrDefault() *instrument.MetricsConfiguration { + if c.Metrics != nil { + return c.Metrics + } + + return &defaultMetrics +} + // WriteWorkerPoolOrDefault returns the write worker pool config or default. -func (c Configuration) WriteWorkerPoolOrDefault() xconfig.WorkerPoolPolicy { +func (c *Configuration) WriteWorkerPoolOrDefault() xconfig.WorkerPoolPolicy { if c.WriteWorkerPool != nil { return *c.WriteWorkerPool } + return defaultWriteWorkerPool } diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 1f9ab9fed6..193ec26aa4 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -52,7 +52,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 0849aa4cd9..51d08e4812 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -99,8 +99,6 @@ db: # Whether new series should be created asynchronously (recommended value # of true for high throughput.) writeNewSeriesAsync: true - # Maximum number of new series that can be created per second. - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms bootstrap: diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index d5cddf3c5a..fe4de58051 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -73,7 +73,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 0d34b5f53a..e76bb65f0e 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -52,7 +52,6 @@ db: gcPercentage: 100 writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms cache: diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 1f9ab9fed6..b40dbaae7d 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -1,76 +1,15 @@ coordinator: - listenAddress: 0.0.0.0:7201 - local: namespaces: - namespace: default type: unaggregated retention: 48h - logging: - level: info - - metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - - tagOptions: - # Configuration setting for generating metric IDs from tags. - idScheme: quoted - db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: config value: m3db_local - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - config: service: env: default_env diff --git a/src/dbnode/runtime/runtime_options.go b/src/dbnode/runtime/runtime_options.go index 8baf6f15d0..aece7e16b5 100644 --- a/src/dbnode/runtime/runtime_options.go +++ b/src/dbnode/runtime/runtime_options.go @@ -33,7 +33,7 @@ const ( DefaultWriteConsistencyLevel = topology.ConsistencyLevelMajority // DefaultReadConsistencyLevel is the default read consistency level - DefaultReadConsistencyLevel = topology.ReadConsistencyLevelMajority + DefaultReadConsistencyLevel = topology.ReadConsistencyLevelUnstrictMajority // DefaultBootstrapConsistencyLevel is the default bootstrap consistency level DefaultBootstrapConsistencyLevel = topology.ReadConsistencyLevelMajority diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 2adc9a52d1..dc19aebd6f 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -167,7 +167,7 @@ func Run(runOpts RunOptions) { cfg = runOpts.Config } - err := cfg.InitDefaultsAndValidate() + err := cfg.Validate() if err != nil { // NB(r): Use fmt.Fprintf(os.Stderr, ...) to avoid etcd.SetGlobals() // sending stdlib "log" to black hole. Don't remove unless with good reason. @@ -175,7 +175,7 @@ func Run(runOpts RunOptions) { os.Exit(1) } - logger, err := cfg.Logging.BuildLogger() + logger, err := cfg.LoggingOrDefault().BuildLogger() if err != nil { // NB(r): Use fmt.Fprintf(os.Stderr, ...) to avoid etcd.SetGlobals() // sending stdlib "log" to black hole. Don't remove unless with good reason. @@ -229,9 +229,9 @@ func Run(runOpts RunOptions) { defer fslock.Release() go bgValidateProcessLimits(logger) - debug.SetGCPercent(cfg.GCPercentage) + debug.SetGCPercent(cfg.GCPercentageOrDefault()) - scope, _, err := cfg.Metrics.NewRootScope() + scope, _, err := cfg.MetricsOrDefault().NewRootScope() if err != nil { logger.Fatal("could not connect to metrics", zap.Error(err)) } @@ -330,7 +330,7 @@ func Run(runOpts RunOptions) { // are constructed allowing for type to be picked // by the caller using instrument.NewTimer(...). timerOpts := instrument.NewHistogramTimerOptions(instrument.HistogramTimerOptions{}) - timerOpts.StandardSampleRate = cfg.Metrics.SampleRate() + timerOpts.StandardSampleRate = cfg.MetricsOrDefault().SampleRate() var ( opts = storage.NewOptions() @@ -392,8 +392,9 @@ func Run(runOpts RunOptions) { SetLimitEnabled(true). SetLimitMbps(cfg.Filesystem.ThroughputLimitMbpsOrDefault()). SetLimitCheckEvery(cfg.Filesystem.ThroughputCheckEveryOrDefault())). - SetWriteNewSeriesAsync(cfg.WriteNewSeriesAsync). - SetWriteNewSeriesBackoffDuration(cfg.WriteNewSeriesBackoffDuration) + SetWriteNewSeriesAsync(cfg.WriteNewSeriesAsyncOrDefault()). + SetWriteNewSeriesBackoffDuration(cfg.WriteNewSeriesBackoffDurationOrDefault()) + if lruCfg := cfg.Cache.SeriesConfiguration().LRU; lruCfg != nil { runtimeOpts = runtimeOpts.SetMaxWiredBlocks(lruCfg.MaxBlocks) } @@ -440,7 +441,8 @@ func Run(runOpts RunOptions) { // FOLLOWUP(prateek): remove this once we have the runtime options<->index wiring done indexOpts := opts.IndexOptions() insertMode := index.InsertSync - if cfg.WriteNewSeriesAsync { + + if cfg.WriteNewSeriesAsyncOrDefault() { insertMode = index.InsertAsync } indexOpts = indexOpts.SetInsertMode(insertMode). @@ -468,7 +470,11 @@ func Run(runOpts RunOptions) { opts = opts.SetRuntimeOptionsManager(runtimeOptsMgr) - policy := cfg.PoolingPolicy + policy, err := cfg.PoolingPolicyOrDefault() + if err != nil { + logger.Fatal("could not get pooling policy", zap.Error(err)) + } + tagEncoderPool := serialize.NewTagEncoderPool( serialize.NewTagEncoderOptions(), poolOptions( @@ -510,28 +516,29 @@ func Run(runOpts RunOptions) { SetMmapReporter(mmapReporter) var commitLogQueueSize int - specified := cfg.CommitLog.Queue.Size - switch cfg.CommitLog.Queue.CalculationType { + cfgCommitLog := cfg.CommitLogOrDefault() + specified := cfgCommitLog.Queue.Size + switch cfgCommitLog.Queue.CalculationType { case config.CalculationTypeFixed: commitLogQueueSize = specified case config.CalculationTypePerCPU: commitLogQueueSize = specified * runtime.NumCPU() default: logger.Fatal("unknown commit log queue size type", - zap.Any("type", cfg.CommitLog.Queue.CalculationType)) + zap.Any("type", cfgCommitLog.Queue.CalculationType)) } var commitLogQueueChannelSize int - if cfg.CommitLog.QueueChannel != nil { - specified := cfg.CommitLog.QueueChannel.Size - switch cfg.CommitLog.Queue.CalculationType { + if cfgCommitLog.QueueChannel != nil { + specified := cfgCommitLog.QueueChannel.Size + switch cfgCommitLog.Queue.CalculationType { case config.CalculationTypeFixed: commitLogQueueChannelSize = specified case config.CalculationTypePerCPU: commitLogQueueChannelSize = specified * runtime.NumCPU() default: logger.Fatal("unknown commit log queue channel size type", - zap.Any("type", cfg.CommitLog.Queue.CalculationType)) + zap.Any("type", cfgCommitLog.Queue.CalculationType)) } } else { commitLogQueueChannelSize = int(float64(commitLogQueueSize) / commitlog.MaximumQueueSizeQueueChannelSizeRatio) @@ -542,14 +549,18 @@ func Run(runOpts RunOptions) { opts = opts.SetSeriesCachePolicy(seriesCachePolicy) // Apply pooling options. - opts = withEncodingAndPoolingOptions(cfg, logger, opts, cfg.PoolingPolicy) + poolingPolicy, err := cfg.PoolingPolicyOrDefault() + if err != nil { + logger.Fatal("could not get pooling policy", zap.Error(err)) + } + opts = withEncodingAndPoolingOptions(cfg, logger, opts, poolingPolicy) opts = opts.SetCommitLogOptions(opts.CommitLogOptions(). SetInstrumentOptions(opts.InstrumentOptions()). SetFilesystemOptions(fsopts). SetStrategy(commitlog.StrategyWriteBehind). - SetFlushSize(cfg.CommitLog.FlushMaxBytes). - SetFlushInterval(cfg.CommitLog.FlushEvery). + SetFlushSize(cfgCommitLog.FlushMaxBytes). + SetFlushInterval(cfgCommitLog.FlushEvery). SetBacklogQueueSize(commitLogQueueSize). SetBacklogQueueChannelSize(commitLogQueueChannelSize)) @@ -663,25 +674,29 @@ func Run(runOpts RunOptions) { if fn := runOpts.StorageOptions.TChanNodeServerFn; fn != nil { tchanOpts = tchanOpts.SetTChanNodeServerFn(fn) } + + listenAddress := cfg.ListenAddressOrDefault() tchannelthriftNodeClose, err := ttnode.NewServer(service, - cfg.ListenAddress, contextPool, tchanOpts).ListenAndServe() + listenAddress, contextPool, tchanOpts).ListenAndServe() if err != nil { logger.Fatal("could not open tchannelthrift interface", - zap.String("address", cfg.ListenAddress), zap.Error(err)) + zap.String("address", listenAddress), zap.Error(err)) } defer tchannelthriftNodeClose() - logger.Info("node tchannelthrift: listening", zap.String("address", cfg.ListenAddress)) + logger.Info("node tchannelthrift: listening", zap.String("address", listenAddress)) + httpListenAddress := cfg.HTTPNodeListenAddressOrDefault() httpjsonNodeClose, err := hjnode.NewServer(service, - cfg.HTTPNodeListenAddress, contextPool, nil).ListenAndServe() + httpListenAddress, contextPool, nil).ListenAndServe() if err != nil { logger.Fatal("could not open httpjson interface", - zap.String("address", cfg.HTTPNodeListenAddress), zap.Error(err)) + zap.String("address", httpListenAddress), zap.Error(err)) } defer httpjsonNodeClose() - logger.Info("node httpjson: listening", zap.String("address", cfg.HTTPNodeListenAddress)) + logger.Info("node httpjson: listening", zap.String("address", httpListenAddress)) - if cfg.DebugListenAddress != "" { + debugListenAddress := cfg.DebugListenAddressOrDefault() + if debugListenAddress != "" { var debugWriter xdebug.ZipWriter handlerOpts, err := placement.NewHandlerOptions(syncCfg.ClusterClient, queryconfig.Configuration{}, nil, iopts) @@ -722,12 +737,12 @@ func Run(runOpts RunOptions) { } } - if err := http.ListenAndServe(cfg.DebugListenAddress, mux); err != nil { + if err := http.ListenAndServe(debugListenAddress, mux); err != nil { logger.Error("debug server could not listen", - zap.String("address", cfg.DebugListenAddress), zap.Error(err)) + zap.String("address", debugListenAddress), zap.Error(err)) } else { logger.Info("debug server listening", - zap.String("address", cfg.DebugListenAddress), + zap.String("address", debugListenAddress), ) } }() @@ -863,23 +878,25 @@ func Run(runOpts RunOptions) { opts = opts.SetBootstrapProcessProvider(bs) // Start the cluster services now that the M3DB client is available. + clusterListenAddress := cfg.ClusterListenAddressOrDefault() tchannelthriftClusterClose, err := ttcluster.NewServer(m3dbClient, - cfg.ClusterListenAddress, contextPool, tchannelOpts).ListenAndServe() + clusterListenAddress, contextPool, tchannelOpts).ListenAndServe() if err != nil { logger.Fatal("could not open tchannelthrift interface", - zap.String("address", cfg.ClusterListenAddress), zap.Error(err)) + zap.String("address", clusterListenAddress), zap.Error(err)) } defer tchannelthriftClusterClose() - logger.Info("cluster tchannelthrift: listening", zap.String("address", cfg.ClusterListenAddress)) + logger.Info("cluster tchannelthrift: listening", zap.String("address", clusterListenAddress)) + httpClusterListenAddress := cfg.HTTPClusterListenAddressOrDefault() httpjsonClusterClose, err := hjcluster.NewServer(m3dbClient, - cfg.HTTPClusterListenAddress, contextPool, nil).ListenAndServe() + httpClusterListenAddress, contextPool, nil).ListenAndServe() if err != nil { logger.Fatal("could not open httpjson interface", - zap.String("address", cfg.HTTPClusterListenAddress), zap.Error(err)) + zap.String("address", httpClusterListenAddress), zap.Error(err)) } defer httpjsonClusterClose() - logger.Info("cluster httpjson: listening", zap.String("address", cfg.HTTPClusterListenAddress)) + logger.Info("cluster httpjson: listening", zap.String("address", httpClusterListenAddress)) // Initialize clustered database. clusterTopoWatch, err := topo.Watch() @@ -926,7 +943,7 @@ func Run(runOpts RunOptions) { // Only set the write new series limit after bootstrapping kvWatchNewSeriesLimitPerShard(syncCfg.KVStore, logger, topo, - runtimeOptsMgr, cfg.WriteNewSeriesLimitPerSecond) + runtimeOptsMgr, cfg.Limits.WriteNewSeriesPerSecond) kvWatchEncodersPerBlockLimit(syncCfg.KVStore, logger, runtimeOptsMgr, cfg.Limits.MaxEncodersPerBlock) }() diff --git a/src/query/api/v1/handler/database/create.go b/src/query/api/v1/handler/database/create.go index 76e433b661..de9e7054f8 100644 --- a/src/query/api/v1/handler/database/create.go +++ b/src/query/api/v1/handler/database/create.go @@ -573,7 +573,7 @@ func defaultedPlacementInitRequest( return nil, errMissingEmbeddedDBConfig } - addr := embeddedDbCfg.ListenAddress + addr := embeddedDbCfg.ListenAddressOrDefault() port, err := portFromEmbeddedDBConfigListenAddress(addr) if err != nil { return nil, err diff --git a/src/query/api/v1/handler/database/create_test.go b/src/query/api/v1/handler/database/create_test.go index f72a4ff540..35f46b2f43 100644 --- a/src/query/api/v1/handler/database/create_test.go +++ b/src/query/api/v1/handler/database/create_test.go @@ -50,8 +50,9 @@ import ( ) var ( - testDBCfg = &dbconfig.DBConfiguration{ - ListenAddress: "0.0.0.0:9000", + listenAddress = "0.0.0.0:9000" + testDBCfg = &dbconfig.DBConfiguration{ + ListenAddress: &listenAddress, } svcDefaultOptions = []handleroptions.ServiceOptionsDefault{ diff --git a/src/query/server/multi_process.go b/src/query/server/multi_process.go index 2ce47b5bb7..dda1c0818b 100644 --- a/src/query/server/multi_process.go +++ b/src/query/server/multi_process.go @@ -78,18 +78,19 @@ func multiProcessRun( } // Set the root scope multi-process process ID. - if cfg.Metrics.RootScope == nil { - cfg.Metrics.RootScope = &instrument.ScopeConfiguration{} + metrics := cfg.MetricsOrDefault() + if metrics.RootScope == nil { + metrics.RootScope = &instrument.ScopeConfiguration{} } - if cfg.Metrics.RootScope.CommonTags == nil { - cfg.Metrics.RootScope.CommonTags = make(map[string]string) + if metrics.RootScope.CommonTags == nil { + metrics.RootScope.CommonTags = make(map[string]string) } - cfg.Metrics.RootScope.CommonTags[multiProcessMetricTagID] = multiProcessInstance + metrics.RootScope.CommonTags[multiProcessMetricTagID] = multiProcessInstance // Listen on a different Prometheus metrics handler listen port. - if cfg.Metrics.PrometheusReporter != nil && cfg.Metrics.PrometheusReporter.ListenAddress != "" { + if metrics.PrometheusReporter != nil && metrics.PrometheusReporter.ListenAddress != "" { // Simply increment the listen address port by instance numbe - host, port, err := net.SplitHostPort(cfg.Metrics.PrometheusReporter.ListenAddress) + host, port, err := net.SplitHostPort(metrics.PrometheusReporter.ListenAddress) if err != nil { return multiProcessResult{}, fmt.Errorf("could not split host:port for metrics reporter: %v", err) @@ -103,7 +104,7 @@ func multiProcessRun( if portValue > 0 { // Increment port value by process ID if valid port. address := net.JoinHostPort(host, strconv.Itoa(portValue+instance-1)) - cfg.Metrics.PrometheusReporter.ListenAddress = address + metrics.PrometheusReporter.ListenAddress = address logger.Info("multi-process prometheus metrics reporter listen address configured", zap.String("address", address)) } diff --git a/src/query/server/query.go b/src/query/server/query.go index 054e7055c1..c67327c669 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -177,7 +177,7 @@ func Run(runOpts RunOptions) { listenerOpts = xnet.NewListenerOptions() ) - logger, err := cfg.Logging.BuildLogger() + logger, err := cfg.LoggingOrDefault().BuildLogger() if err != nil { // NB(r): Use fmt.Fprintf(os.Stderr, ...) to avoid etcd.SetGlobals() // sending stdlib "log" to black hole. Don't remove unless with good reason. @@ -209,7 +209,7 @@ func Run(runOpts RunOptions) { } prometheusEngineRegistry := extprom.NewRegistry() - scope, closer, reporters, err := cfg.Metrics.NewRootScopeAndReporters( + scope, closer, reporters, err := cfg.MetricsOrDefault().NewRootScopeAndReporters( instrument.NewRootScopeAndReportersOptions{ PrometheusExternalRegistries: []instrument.PrometheusExternalRegistry{ { @@ -488,7 +488,8 @@ func Run(runOpts RunOptions) { logger.Fatal("unable to register routes", zap.Error(err)) } - srv := &http.Server{Addr: cfg.ListenAddress, Handler: handler.Router()} + listenAddress := cfg.ListenAddressOrDefault() + srv := &http.Server{Addr: listenAddress, Handler: handler.Router()} defer func() { logger.Info("closing server") if err := srv.Shutdown(context.Background()); err != nil { @@ -496,10 +497,10 @@ func Run(runOpts RunOptions) { } }() - listener, err := listenerOpts.Listen("tcp", cfg.ListenAddress) + listener, err := listenerOpts.Listen("tcp", listenAddress) if err != nil { logger.Fatal("unable to listen on listen address", - zap.String("address", cfg.ListenAddress), + zap.String("address", listenAddress), zap.Error(err)) } if runOpts.ListenerCh != nil { @@ -509,7 +510,7 @@ func Run(runOpts RunOptions) { logger.Info("starting API server", zap.Stringer("address", listener.Addr())) if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { logger.Fatal("server serve error", - zap.String("address", cfg.ListenAddress), + zap.String("address", listenAddress), zap.Error(err)) } }() From 6551b983829469664128b4463e1944cdb773a000 Mon Sep 17 00:00:00 2001 From: teddywahle <69990143+teddywahle@users.noreply.github.com> Date: Sat, 7 Nov 2020 00:49:58 -0500 Subject: [PATCH 33/47] [query] Remove single series error from M3 --- src/query/graphite/native/builtin_functions.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/query/graphite/native/builtin_functions.go b/src/query/graphite/native/builtin_functions.go index 4528c07c1d..f757ddb528 100644 --- a/src/query/graphite/native/builtin_functions.go +++ b/src/query/graphite/native/builtin_functions.go @@ -1031,11 +1031,6 @@ func asPercent(ctx *common.Context, input singlePathSpec, total genericInterface toNormalize = input.Values tf = func(idx int, _ *ts.Series) float64 { return totalBySum(normalized, idx) } } else { - // check total is a single-series list and normalize all of them - if total.Len() != 1 { - err := errors.NewInvalidParamsError(errors.New("total must be a single series")) - return ts.NewSeriesList(), err - } if len(nodes) > 0 { // group the series by specified nodes and then sum those groups groupedTotal, err := groupByNodes(ctx, input, "sum", nodes...) From 55e6e44bd28ee9f80c8f6120147c15daa82193f8 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sat, 7 Nov 2020 13:27:08 -0500 Subject: [PATCH 34/47] Bump go to 1.14 (#2853) --- .buildkite/pipeline.yml | 8 ++++---- Makefile | 4 ++-- docker-compose.yml | 2 +- docker/m3aggregator/Dockerfile | 2 +- docker/m3collector/Dockerfile | 2 +- docker/m3coordinator/Dockerfile | 2 +- docker/m3dbnode/Dockerfile | 2 +- docker/m3dbnode/Dockerfile-setcap | 2 +- docker/m3query/Dockerfile | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b435db97ea..4d8569d0e8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,7 +14,7 @@ steps: command: make clean install-vendor-m3 test-all-gen env: CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.13.x + GIMME_GO_VERSION: 1.14.x plugins: gopath-checkout#v1.0.1: import: github.com/m3db/m3 @@ -56,7 +56,7 @@ steps: parallelism: 2 env: CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.13.x + GIMME_GO_VERSION: 1.14.x plugins: gopath-checkout#v1.0.1: import: github.com/m3db/m3 @@ -66,7 +66,7 @@ steps: parallelism: 1 env: CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.13.x + GIMME_GO_VERSION: 1.14.x plugins: gopath-checkout#v1.0.1: import: github.com/m3db/m3 @@ -99,7 +99,7 @@ steps: command: make clean install-vendor-m3 docs-test env: CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.13.x + GIMME_GO_VERSION: 1.14.x plugins: gopath-checkout#v1.0.1: import: github.com/m3db/m3 diff --git a/Makefile b/Makefile index f4e3531e55..92943d6670 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,8 @@ GO_BUILD_LDFLAGS_CMD := $(abspath ./scripts/go-build-ldflags.sh) GO_BUILD_LDFLAGS := $(shell $(GO_BUILD_LDFLAGS_CMD) LDFLAG) GO_BUILD_COMMON_ENV := CGO_ENABLED=0 LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 $(GO_BUILD_COMMON_ENV) -# GO_RELEASER_DOCKER_IMAGE is latest goreleaser for go 1.13 -GO_RELEASER_DOCKER_IMAGE := goreleaser/goreleaser:v0.127.0 +# GO_RELEASER_DOCKER_IMAGE is latest goreleaser for go 1.14 +GO_RELEASER_DOCKER_IMAGE := goreleaser/goreleaser:v0.141.0 GO_RELEASER_RELEASE_ARGS ?= --rm-dist GO_RELEASER_WORKING_DIR := /go/src/github.com/m3db/m3 diff --git a/docker-compose.yml b/docker-compose.yml index 5097f20e75..36aa185329 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ app: - image: golang:1.13-stretch + image: golang:1.14-stretch volumes: - .:/go/src/github.com/m3db/m3 - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent diff --git a/docker/m3aggregator/Dockerfile b/docker/m3aggregator/Dockerfile index b637388c29..a423bc03ac 100644 --- a/docker/m3aggregator/Dockerfile +++ b/docker/m3aggregator/Dockerfile @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps diff --git a/docker/m3collector/Dockerfile b/docker/m3collector/Dockerfile index f0e55b9930..e19f4da382 100644 --- a/docker/m3collector/Dockerfile +++ b/docker/m3collector/Dockerfile @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps diff --git a/docker/m3coordinator/Dockerfile b/docker/m3coordinator/Dockerfile index 675183b68d..eba6c1bf7e 100644 --- a/docker/m3coordinator/Dockerfile +++ b/docker/m3coordinator/Dockerfile @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps diff --git a/docker/m3dbnode/Dockerfile b/docker/m3dbnode/Dockerfile index de6ca6e830..d85a4d7138 100644 --- a/docker/m3dbnode/Dockerfile +++ b/docker/m3dbnode/Dockerfile @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps diff --git a/docker/m3dbnode/Dockerfile-setcap b/docker/m3dbnode/Dockerfile-setcap index a2b7463d27..49d8bffafe 100644 --- a/docker/m3dbnode/Dockerfile-setcap +++ b/docker/m3dbnode/Dockerfile-setcap @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps diff --git a/docker/m3query/Dockerfile b/docker/m3query/Dockerfile index 58a6d8fe8a..bd548db079 100644 --- a/docker/m3query/Dockerfile +++ b/docker/m3query/Dockerfile @@ -1,5 +1,5 @@ # stage 1: build -FROM golang:1.13-alpine3.11 AS builder +FROM golang:1.14-alpine3.11 AS builder LABEL maintainer="The M3DB Authors " # Install deps From 3f81fe32bb080bf5e5f28f5a498885ddda4b78b7 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sun, 8 Nov 2020 12:19:07 -0500 Subject: [PATCH 35/47] [aggregator] Process campaign state without waiting for first campaign check interval (#2855) --- src/aggregator/aggregator/election_mgr.go | 31 ++++++++----- .../aggregator/election_mgr_test.go | 45 ++++++++++++++++--- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/aggregator/aggregator/election_mgr.go b/src/aggregator/aggregator/election_mgr.go index 8c685cf2bc..7f3d0b55e8 100644 --- a/src/aggregator/aggregator/election_mgr.go +++ b/src/aggregator/aggregator/election_mgr.go @@ -579,19 +579,28 @@ func (mgr *electionManager) checkCampaignStateLoop() { defer ticker.Stop() for { + select { + case <-mgr.doneCh: + return + default: + } + + enabled, err := mgr.campaignIsEnabledFn() + if err != nil { + mgr.metrics.campaignCheckErrors.Inc(1) + + continue + } + newState := newCampaignState(enabled) + currState := mgr.campaignState() + if currState == newState { + continue + } + mgr.processCampaignStateChange(newState) + + // block on ticker or exit signal select { case <-ticker.C: - enabled, err := mgr.campaignIsEnabledFn() - if err != nil { - mgr.metrics.campaignCheckErrors.Inc(1) - continue - } - newState := newCampaignState(enabled) - currState := mgr.campaignState() - if currState == newState { - continue - } - mgr.processCampaignStateChange(newState) case <-mgr.doneCh: return } diff --git a/src/aggregator/aggregator/election_mgr_test.go b/src/aggregator/aggregator/election_mgr_test.go index fbb5bbbc75..d88dc70842 100644 --- a/src/aggregator/aggregator/election_mgr_test.go +++ b/src/aggregator/aggregator/election_mgr_test.go @@ -134,6 +134,9 @@ func TestElectionManagerOpenSuccess(t *testing.T) { return make(chan campaign.Status), nil }). AnyTimes() + leaderService.EXPECT(). + Resign(gomock.Any()). + AnyTimes() opts := testElectionManagerOptions(t, ctrl).SetLeaderService(leaderService) mgr := NewElectionManager(opts).(*electionManager) @@ -245,6 +248,7 @@ func TestElectionManagerResignLeaderServiceResignError(t *testing.T) { } func TestElectionManagerResignTimeout(t *testing.T) { + t.Parallel() ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -272,14 +276,16 @@ func TestElectionManagerResignTimeout(t *testing.T) { } func TestElectionManagerResignSuccess(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() var ( - statusCh = make(chan campaign.Status, 2) + statusCh = make(chan campaign.Status, 1) mgr *electionManager ) @@ -290,7 +296,10 @@ func TestElectionManagerResignSuccess(t *testing.T) { leaderService.EXPECT(). Resign(gomock.Any()). DoAndReturn(func(string) error { - statusCh <- campaign.Status{State: campaign.Follower} + select { + case statusCh <- campaign.Status{State: campaign.Follower}: + default: + } return nil }). AnyTimes() @@ -300,6 +309,7 @@ func TestElectionManagerResignSuccess(t *testing.T) { campaignOpts = campaignOpts.SetLeaderValue(leaderValue) opts := testElectionManagerOptions(t, ctrl). SetCampaignOptions(campaignOpts). + // SetCampaignStateCheckInterval(1 * time.Second). SetLeaderService(leaderService) i := placement.NewInstance().SetID("myself") opts.PlacementManager().(*MockPlacementManager). @@ -319,9 +329,20 @@ func TestElectionManagerResignSuccess(t *testing.T) { require.NoError(t, mgr.Open(testShardSetID)) require.NoError(t, mgr.Resign(ctx)) - time.Sleep(time.Second) + + var mgrState electionManagerState + for i := 0; i < 10; i++ { + mgr.RLock() + mgrState = mgr.state + mgr.RUnlock() + if mgr.ElectionState() == FollowerState && mgrState == electionManagerOpen { + break + } + time.Sleep(100 * time.Millisecond) + } + require.Equal(t, FollowerState, mgr.ElectionState()) - require.Equal(t, electionManagerOpen, mgr.state) + require.Equal(t, electionManagerOpen, mgrState) require.NoError(t, mgr.Close()) } @@ -346,6 +367,8 @@ func TestElectionManagerCloseSuccess(t *testing.T) { } func TestElectionManagerCampaignLoop(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -380,6 +403,7 @@ func TestElectionManagerCampaignLoop(t *testing.T) { campaignOpts = campaignOpts.SetLeaderValue(leaderValue) opts := testElectionManagerOptions(t, ctrl). SetCampaignOptions(campaignOpts). + SetCampaignStateCheckInterval(100 * time.Millisecond). SetLeaderService(leaderService) i := placement.NewInstance().SetID("myself") opts.PlacementManager().(*MockPlacementManager). @@ -479,6 +503,8 @@ func TestElectionManagerCampaignLoop(t *testing.T) { } func TestElectionManagerVerifyLeaderDelayWithValidLeader(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -544,6 +570,8 @@ func TestElectionManagerVerifyLeaderDelayWithValidLeader(t *testing.T) { } func TestElectionManagerVerifyLeaderDelayWithLeaderNotInPlacement(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -603,6 +631,8 @@ func TestElectionManagerVerifyLeaderDelayWithLeaderNotInPlacement(t *testing.T) } func TestElectionManagerVerifyLeaderDelayWithLeaderOwningDifferentShardSet(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -667,6 +697,8 @@ func TestElectionManagerVerifyLeaderDelayWithLeaderOwningDifferentShardSet(t *te } func TestElectionManagerVerifyWithLeaderErrors(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -815,6 +847,8 @@ func TestElectionManagerVerifyCampaignDisabled(t *testing.T) { } func TestElectionManagerCheckCampaignStateLoop(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -834,6 +868,7 @@ func TestElectionManagerCheckCampaignStateLoop(t *testing.T) { campaignOpts = campaignOpts.SetLeaderValue(leaderValue) opts := testElectionManagerOptions(t, ctrl). SetCampaignOptions(campaignOpts). + SetCampaignStateCheckInterval(100 * time.Millisecond). SetLeaderService(leaderService) mgr := NewElectionManager(opts).(*electionManager) iterCh := make(chan enabledRes) From fa680ead843d786411ab7d357d3f614a120709ec Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sun, 8 Nov 2020 13:14:59 -0500 Subject: [PATCH 36/47] Update ci-scripts to correct coverage tracking (#2854) --- .ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci b/.ci index 96907c2669..15209040a6 160000 --- a/.ci +++ b/.ci @@ -1 +1 @@ -Subproject commit 96907c2669187b166eead31d9e9a5bc4fcbb9b52 +Subproject commit 15209040a6432a3280c1e2ed2d55ebd520ebe723 From 4071759bedb5cc3dda07c69d307ffa471be7cef8 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sun, 8 Nov 2020 21:53:30 -0500 Subject: [PATCH 37/47] Cleanup m3nsch leftovers (#2856) --- docker/images.json | 4 - integrations/grafana/m3nsch_dashboard.json | 499 --------------------- 2 files changed, 503 deletions(-) delete mode 100644 integrations/grafana/m3nsch_dashboard.json diff --git a/docker/images.json b/docker/images.json index 5a7feb1bb8..ecea5f3d2c 100644 --- a/docker/images.json +++ b/docker/images.json @@ -17,10 +17,6 @@ "name": "m3dbnode", "tag_suffix": "setcap" }, - "m3nsch": { - "dockerfile": "docker/m3nsch/Dockerfile", - "name": "m3nsch" - }, "m3query": { "dockerfile": "docker/m3query/Dockerfile", "name": "m3query" diff --git a/integrations/grafana/m3nsch_dashboard.json b/integrations/grafana/m3nsch_dashboard.json deleted file mode 100644 index 32e48ac9e7..0000000000 --- a/integrations/grafana/m3nsch_dashboard.json +++ /dev/null @@ -1,499 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.2.4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1550848579805, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "panels": [], - "title": "Write Requests", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(agent_write_success{instance=~\"$instance\"}[$step])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "writes-success-{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Write Success / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(agent_write_errors{instance=~\"$instance\"}[$step])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "writes-errors-{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Write Errors / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "agent_write_success_latency{instance=~\"$instance\",quantile=\"0.99\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "write-success-{{instance}}-p99", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Write Success P99", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "agent_write_errors_latency{instance=~\"$instance\",quantile=\"0.99\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "write-errors-{{instance}}-p99", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Write Errors P99", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "instance", - "options": [], - "query": "label_values(agent_write_success,instance)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "tags": [], - "text": "1m", - "value": "1m" - }, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "step", - "options": [ - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - } - ], - "query": "30s,1m,5m,10m", - "type": "custom" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "M3nsch", - "uid": "1gePBprmz", - "version": 6 -} From 48f781fce7fdc6a681a83fec09fdbf7003267c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Mon, 9 Nov 2020 09:00:15 +0200 Subject: [PATCH 38/47] [dbnode] Use bits.LeadingZeros64 to improve encoder performance (#2857) Co-authored-by: Rob Skillington --- src/dbnode/encoding/encoding.go | 19 +--- src/dbnode/encoding/encoding_test.go | 6 ++ .../encoding/m3tsz/encoder_benchmark_test.go | 102 ++++++++++++++++++ 3 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 src/dbnode/encoding/m3tsz/encoder_benchmark_test.go diff --git a/src/dbnode/encoding/encoding.go b/src/dbnode/encoding/encoding.go index 4d65f5e921..f6623124ec 100644 --- a/src/dbnode/encoding/encoding.go +++ b/src/dbnode/encoding/encoding.go @@ -22,25 +22,16 @@ package encoding import "math/bits" -// Bit is just a byte +// Bit is just a byte. type Bit byte -// NumSig returns the number of significant values in a uint64 +// NumSig returns the number of significant bits in a uint64. func NumSig(v uint64) uint8 { - if v == 0 { - return 0 - } - - numLeading := uint8(0) - for tmp := v; (tmp & (1 << 63)) == 0; tmp <<= 1 { - numLeading++ - } - - return uint8(64) - numLeading + return uint8(64 - bits.LeadingZeros64(v)) } // LeadingAndTrailingZeros calculates the number of leading and trailing 0s -// for a uint64 +// for a uint64. func LeadingAndTrailingZeros(v uint64) (int, int) { if v == 0 { return 64, 0 @@ -51,7 +42,7 @@ func LeadingAndTrailingZeros(v uint64) (int, int) { return numLeading, numTrailing } -// SignExtend sign extends the highest bit of v which has numBits (<=64) +// SignExtend sign extends the highest bit of v which has numBits (<=64). func SignExtend(v uint64, numBits uint) int64 { shift := 64 - numBits return (int64(v) << shift) >> shift diff --git a/src/dbnode/encoding/encoding_test.go b/src/dbnode/encoding/encoding_test.go index c48ad981b2..a81a86ae08 100644 --- a/src/dbnode/encoding/encoding_test.go +++ b/src/dbnode/encoding/encoding_test.go @@ -21,11 +21,17 @@ package encoding import ( + "math" "testing" "github.com/stretchr/testify/require" ) func TestNumSig(t *testing.T) { + require.Equal(t, uint8(0), NumSig(uint64(0))) + require.Equal(t, uint8(1), NumSig(uint64(1))) require.Equal(t, uint8(4), NumSig(uint64(12))) + require.Equal(t, uint8(63), NumSig(uint64(math.MaxUint64>>1))) + require.Equal(t, uint8(64), NumSig(uint64(math.MaxUint64))) + require.Equal(t, uint8(64), NumSig(uint64(math.MaxUint64-1))) } diff --git a/src/dbnode/encoding/m3tsz/encoder_benchmark_test.go b/src/dbnode/encoding/m3tsz/encoder_benchmark_test.go new file mode 100644 index 0000000000..67c62b4660 --- /dev/null +++ b/src/dbnode/encoding/m3tsz/encoder_benchmark_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package m3tsz + +import ( + "bytes" + "encoding/base64" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/m3db/m3/src/dbnode/encoding" + "github.com/m3db/m3/src/dbnode/ts" + xtime "github.com/m3db/m3/src/x/time" +) + +var sampleSeriesBase64 = []string{ + "FiYqRnIdAACAQEAAAAArkizADVrDlnvgAATiGAAEbAABZgABkwAA6DFXAxnaGOwDF2ON7Yw85trFGksvYiyRjTFW3MeYs21wLHm9t/YkxtjbHW5vCYi6JwTF2LMcYsGI2DGdTRBjsCxRi7bHdsRZI2ZjDdGQsfbs15ijGHosPYqxNjjPGnMcYu29jbJmusVY03FibeGkMYY8xVizVHHsXY+3BjTR2NMYcE2ti7V2yMZb63hi7dmdMYdoxpizgGxMWa805ljgGMsVY4zRiLiHWslZo11lLOGLMdY61Zkjd2uMRZi1BljI2ostbo1hmDfHasVZUytjTeWOshZK3BjTdGtsWYwxdjwYjgMZpNwzLKM8+btsqGOwjHGMNubIxtnTVWVt1bUxRtLWmWtnY+x1nLU2YtjcuzJw7VWbMfYu0RjLVWbM6aY4lpjT2LtVaS0NqTGGJNeYq3torFWMNJaS1ZrTRWpuCYw1xjLFmItCaExJkDWGZMWZg6xjLMGLtiZmxps7EWLNlYw6NjzFmLtvZaxhi7GGNBiPAxmK8DRM0yj8uq2TKMk0DZOu+rPMsyjQumGOxTgGMNzaaxVrLEWLMUZk0xoDy2QN3Y8yNvLNGmM0boxRtrxGNMcY20dy7G2fM2bqyBjrXmHNyY4xlvzGWJsXcIxdt7H2LtIY2xRq7gGJsbZoxRiTVWVtvaey92LdGKMeYsxoMR+GM9WgZcMdsWKNrcIxNibl2KMaY0x5mTOWOvecYxRuDbGLsubWxJpjaWKsebExZv7JGKsucAxVu7HGOMfbkxdtjdGLMZY8xBkjH2Kt1d2xVtzIGLuCYyyBjTJ2KstbWxVtDbmMMzY6xF4bPWJtxdgxJvrJWMsdaGxhuzTWJs1egxRt7ZmItNYuxRpzFmOtvdyw9kTZ2LtzdaxZiTV2LsabYxJmTXWJtzZCx5pTH2Lt4cQxdtTiWNNea4xNn7imLtccaxVjTZmLMYYuxZnDSmNM0euxVmjU2KtwcWxRjrj2JsbdsxhjjHWNhiOAxW9rhjOwMdl2LN3aczRjbsmOOCbkxhkDa2LN3Zo1xtjGGMtxbexNmLJWJsZbQ19jDU2LNydwxZnLIGONwbI1xuTNGLNqYwxNnbVmQMdcg15uDF2NtKbaxdq7SWKtqa015jbbmNMib2x9mrHmMtxZA1htrWmLNzZGxNoLQmONzbA1drbGmJt0ZCxRjLIWJt0Y41lsDNWJtiaqxFjzF2OuEbk1ltjRGKNYZUxRtjI2MN/eI11vbe2Jsob4xljrJmKttaM19j7HGKuEaOxJkLdmJOIcW1hmLbWNMvY6xZmTHmMs9b82Fk7TmKM7cKxtijW2LMuYy2BpLQ2NNacOxpjbg2OODaSxp4LVmJtfbux1vcAA", // nolint:lll + "FiYqRnIdAACAQEAAAAArkizADAfgAATiCSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSTAAA=", // nolint:lll + "FiYqRnIdAACAQEAAAABDnTBYE/Bho3fYmHbfAAAnEMAB+lf3Y8nesGNjT6sqdklzsPFHkkSBobeKPJIkDQy3ijySJA0MlgE3WlChWbTdaUKFZkudjtB7gub4oltoPcFzfFEkkksFUt2Tfa6Fqpbsm+10lzsPqTuES/mJJJLpW9i+c6zi+SW7F851nF9uxfOdZxfLdi+c6zi+SSXOw8DYDYkTFt4GwGxImLLeBsBsSJi28DYDYkTFkulb2L5zrOL5JdC/dMVuc3q9t0xW5zer23TFbnN6vLbpitzm9XtumK3Ob1eW3TFbnN6vJLbpitzm9Xufh7R1X1eVLLJJaw/0a8y0ktYf6NebSS1h/o15lpJaw/0a82klrD/RrzLSS1h/o15lz8PaOq+rypZJYpkiO1dsA1MkR2rtkunPPlZJttpJc/D/fBmnRHWWSS1h/o15lpJaw/0a82klrD/RrzLSS1h/o15kloYescF8rGhh6xwXytz8P1pjNBhIbfrTGaDCQ/AAAnZn///2ERxWSw4LulSvs8twXdKlfZ7cF3SpX2e6F98YLEnhMbXxgsSeExtfGCxJ4TGWvjBYk8JjJJLn3cL98PJ8jbhfvh5PkZLoX3sr7uILjlr2V93EFxySS593C/fDyfI24X74eT5G6F97K+7iC47Xsr7uILjteyvu4guOWvZX3cQXHJJJa9lfdxBcdr2V93EFx3Pw9tAaypmht7aA1lTNDLe2gNZUzQyS3toDWVM0MktDJI57e/ac7HxxmbkR/pJYIWOVrdpMJJJaFjla3aTLQscrW7SbQscrW7SZaFjla3aTJJJaFjla3aTdC+AoWxZUHMtRGb6NHgwWojN9GjwZJaiM30aPBkudh5g7Stcc3JJbzB2la45u3mDtK1xzckksJxZZl2POLTiyzLsedpxZZl2POWnFlmXY85JJc7DzB2la45u3mDtK1xzdhOLLMux5xdK3UGgGFJIS2oNAMKSQkujZOLLMux5yXOw8wdpWuObt5g7Stcc3LeYO0rXHN28wdpWuObksEiGQkVkWJJLo3X2kSbTyRdCywmW9XXelz8OPQTV9E75bj0E1fRO+Szv8H06YXklzsf957cWnANv957cWnANv957cWnAMklv957cWnAMlhn8hhg0Dol0L4gCZVqxQ3Pw49BNX0Tvtx6CavonfJZEATKtWKGSWiAJlWrFDLRAEyrVihtEATKtWKG59+bUdK+4kLZtR0r7iQkls2o6V9xIS2bUdK+4kLZtR0r7iQls2o6V9xIWzajpX3EhLn4dmc/ehX1W7M5+9Cvqkkt2Zz96FfVbszn70K+qW7M5+9Cvqt2Zz96FfVYLZCxIudM1shYkXOmS5+HZnP3oV9Ukt2Zz96FfVLdmc/ehX1W7M5+9CvqtMWFPI9/rJJJYsdtxTlpY1jtuKctLLRg2ocIN1owbUOEG66F3R+2WXEH26P2yy4g+SSS3R+2WXEHy3R+2WXEH26P2yy4g+3R+2WXEH2Rg2ocIN0SSWPcAhr1yze4BDXrlkujZGDahwg3WjBtQ4QbpdCnzVjlG88kkklz7KKnPV58+602dsW5NrbO2Lcm1kklz7KKnPV58+1FTnq8+fJdabO2Lcm1ls7YtybWSSSS59lFTnq8+faipz1efPlqKnPV58+1FTnq8+fJJLrTZ2xbk2stnbFuTa2zti3Jtbo3f04J9i5nZEamPsK4pLo2wbtt7vTWS59MzEoWOlrTMShY6Wklz8PEbYJKYAbnYY8FgdCtcyWH3EI5E2HN9xCORNhyXOwx4LA6Fa5kuhfokiJV00GS59MzEoWOlpJLTMShY6WkuhfokiJV00GSSWG4TfVnMCbcJvqzmBLbhN9WcwLbhN9WcwJbcJvqzmBLnY+85UiQZUpb3nKkSDKlJYG4TfVnMCJdC+kcG+4Y9stSODfcMe2XNxxwbSeDL2XNy6dODPdz6pJJLCdLyOrAioktOl5HVgRVz7ultcsuJ3kkl0bsl+P9BB4kuhZtpdeDGBS6F9/6WHcIbJJZkvx/oIPEl0LZGbL+mLngLftIzZf0xc8B0jNl/TFzl0L/UkMAGrbSSSWZL8f6CDxLZL8f6CDxbJfj/QQeJdC+6q01qmjEkklzsdKzCAxSsUtpWYQGKVit3MU5BmXyliJeKJtHI8kks9H3goTte3o+8FCdry3o+8FCdr29H3goTteXQviXiibRyPJLeaMpHsCQFvNGUj2BLeaMpHsCS3mjKR7AlvNGUj2BJbzRlI9gSSSXRu0uvhH+2y2l18I/22S2l18I/22W0uvhH+2yW0uvhH+2ySSSS52H18ZprwH9vr4zTXgP5b6+M014D+SSSSSXRuii6kmXyCSSS0UXUky+QS0UXUky+QSXNy0u9Sg9bxrZzy5yJx+gl0L+SxCJdwZS3JYhEu4MrcliES7gyluSxCJdwZSSSSS3JYhEu4MpbksQiXcGUktyWIRLuDKSyLVs+rG4paLVs+rG4rRatn1Y3FLRatn1Y3FJaLVs+rG4pJJc/CxW6lPyeuSWhvJm7oR4Ekkl0L8VpabU7JWxWlptTslLYrS02p2SksWdn1v2KcS6NfcU2S5Ky3cU2S5KyS6VlnZ9b9inLWdn1v2KcktZ2fW/YpySWs7PrfsU7Wdn1v2KctZ2fW/YpyXPw8jCvW8FNtk3WHmchTJY7imyXJWQW7imyXJWW7imyXJWSSW7imyXJWSS3cU2S5KyW7imyXJWW7imyXJWS3cU2S5KyS6dV0H/Ok0skuhZXSD4UAdy6Ft3YTNCVqtd2EzQlapa7sJmhK1YAA", // nolint:lll + "FiYqRnIdAACAQEAAAAAWlSx4Dadc6Q14AAE4hgAGQBgAGP9gAGTpgAGFcMxyJvHg8gDyAvFs8e7yAPIC8f7yAPFu8fLyCvH28gL2yu0G8gDyAPII8f7yAPB88YryAQHx9X7yEQHx+vH68U7x+vH+8gLyAPGQ8YTyAPIA8fzyBPDC8iLyAvII8e7yBvGw8ijyCvHw8gTyAvHi8ezx4vIA8hDx9vHw8e7yAPH68gDyBvGA8cDyAPIA8gDyCvE68PTx+vIK8fbyCPBE25jqmPIW8fbyAPCk8gTx/vIC8gjx9vAe7VrpJPIG8gDx+u/i8gjyBvH48g8W8ftU8S8W8WTyAvH+8grx8PD+8erx9PH88f7yCPEu8fryAvH48gbx/Pco7K7x+PIG8gD26u0E8gDyAPIA8grx9vIK8cbyAPIA8gL3GuoU8cDyEPH28gDx+vF08dzx+vIG8gDyAPF88ibx1vIK8fzyAPE68b7yAvH+8fryBvH48g7x+PH88jTx0vHi72bx8PH+8gLyBvA48fzyAvIA8f7yAPIW8eryAPIA8gDyBvGy8fLx+vII8fjyAPHc8EryBvHy8gLx7PHe8XTyAPH68gbyCu+28fTx+PIA8gbyAPCY8fLx/PIE8gbx9PGU1/zt7PH48hLx7vHE8gDyAPIG8gDx+vVU7Sjx/vIC8f7x+vGC8frx9vIE8gLyAPEc8hLx+PH68gjx/vH68f7x+vIG8gDx+vD48fryBPIA8grx+PEi8fjx+PIG8grx9vCY3tTx+PIA8grx8vHu8ezyAPH+8gDx/vJg8PryAPIC8fjyAPE68aLyAvII8fjyAPHY8YzyAPIA8gDyAPDk8fbx8PIG8fryBvEo8ijx9vH88gzyAPHA8ijx6vIU8gDx+vFRAfHvfvH/AfIK8f7yAvHy8gzyAPIK8fDyDvDo8WbyHPIE8fjyDvDq6hbyAPH68gbyAPEw8fryAPIA8grx6PHK8fLyAPIG8gbx9PFC8fLyAQPyAXvyAQHyCwHxauyQ8gDyAPH48gjyCvHw8hDx9vH68gbwqOlO7NLx+PII8fDxZvGy8e7yAwHyAX7yBwHyAPG+8fTyAvH48g0B8OV+8cMB8gDx+vIG8frx0vGU8gbx+PII8gDzAPAo8f7yCPH88fjw3vGM8gTx8vIG8gDyHvHI8fsB8gd+8gMB8gjw4PH28fryAPIG8gjxIvIU8fbx+vIG8gDw1uxE8fryBvIC8f7xpPHY8gbyAPIA8gEC8Mt88UcC8hLx+PIBBfIHdvEhBfHQ8gDx+vIC8gTxkPH+8gLx/vIG8fsB8el+8WkB8fDyCvIQ8fbx/PHw8gbyAvH+8gD1bO0s8gLyBvIA8gTxavIA8gDyAvII8fDxKPEM8fDyAPIG8fzw0vHq8gLx/vIA8gD2Ju0W8fryEPHw8gbxgPH28gbyKPHO8gzxzN0s6q7yBvII9tTtDvIG8gbyAPIC8f7xaPIA8fDyEPH28grx+PFo8fryDPH68frxhPIC8gbyAPIB4AABWoM1ggP///qWNF28AAArU2PFsgDY/WyCNj/bWtAdYnv2yCQHY/myCQXbebtrM0F2OdsfDZAGyANkDbEU2LZsgDY/WyCNkBbH+2PlsgbZAGyCtsKa1u2P1sgbZAGyAtj8bGu2PtsgDY/WyANkDgOx237Y+IDsgDZBmx/Ntuaq+zvVsirY+2yAtkAbFC2PhsfbZAGx+tkAbE42QBsf7ZAGyCtj7bKu2JhsgbY/WyBtj9a5O2QdsfzY/2yCNj9bGu2O1sfzY/2yBtkAbG02MZsgDZA2x+tkDbHo2QBsgbY/WyBtkAbGm2CxsgrY+2x+tj/bFk2EVsfrY/2yCNkBbIg2O9sfrZA2x+tkAbEG1UlsfrZA2yANkBbIUztpsQzZAWx/tj9bE22QNsgbZBWx9tkBbEs2N9sgLY/2yANkDbHO2QdsgrY+2yANkFbFlAdjrv2yAQHZAGyCtj+a9W2QJsgDZAWx/tkAbGw2QFsgbZAGx+tkDbHu2PZsgDZAGx+NkEbGq2FVsgjY/Gx+QXZA7tseEF2QlsgDZAGx+tkWguy7XbYcoLsfkAA=", // nolint:lll + "FiYqRnIdAACAQEAAAAAnPgFYA+AABOIJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJPFcvVHPFcpJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJPgAAGhz4AABno////L/JJJJJJJJJgAA==", // nolint:lll + "FiYqRnIdAACAQEAAAAAWlSx4DYa+fHfgAATiGAAgoYABJ1gABamAAJ9DLWSWyU46nCsw5GX05SG/Y2OWyCwqmgq2cGuK2q25IwqGLg1gmuS2uWlAypm0+2QIDrJX7YLgOxGyh2lo3k2/a1eW3Gwqmdq3BWvU4I2/6w1GXS3eXH01VGm0woGYu0jXEC02G1mxlGTY47Gyi18WpMwbGjW2zm0Kz8Wx2wy2fY1wmoO1wHGiwy2XO3fm3O1RmzEwKWCAzp2382R2/cwj2fw21nE81KWtmwVGGoxKG5Y0/moSxTmVg1lmri2P4tk9VTEhi2RA1OG7c1vmsKxnmWYzy238202rgwCmuS2C2wU0h252xbWQw4EWtO4dHA4wNWhM3hWc81fG+oyLGqS26G5s2dWpIxTmZK3/m0I2129CwX2kwzgGxY1EnBMxKmtG5Bmxy2D2rKxa2fS1GW1+11G64xNmmW1DG1e1Im86wImlA1rWpg2k20gwPGbO2SXI625nCQwqWgM1BXAw2nHMOxQmqA1x2ps2+2qUxbWZU2tGvi2s2tiwsmYo1Hmqo3bW6swhmSC2dWnY0rW7owRWgW2CWi62Qm5kwPmGKzyWuC1UHPAw5m9M3jm524Vmk8w7WSe2XHDC4TW7KwbGqE3BGsG1S3AQxpWXEz6Gto1MmyGwbmZU2H2+419mxMwHWcAzWHCO3AGsWxNWm82jW3W1KmxEwLmJQ2825wyzGsQw0Gae3v2oI0sm5ywa2Ue1ymve4FHEoxRWfQ38m2I2vWpmxI2ee1mWoe1R2yQw9Wos1AGvs0nW7gyAWZu2BGfg1+2sGxiGkW2GXlg3IHCqx4IDoyX7e5gOsg0/mruwQGhG1eW8c3Gm8Kws2ZS4ZnCy30WjKwX2UI2Bmui1oHCeyRmxk19mrI1W25Uw5WWo2HWvm2u2nuwrmr8zXoHqg3vgygOzlAcD9ikNTRxANflu2sYFjFtSJqRNJhrmsH9hTMnBtvNW1vKMK9i7t0huMQHWYv3DvAcXpqktctxBuGhruwHCxv2PPAdTRr0NHFtbsM1kytcZsaN7ZsNMN9oYNThsdtw9o9sLplmuHBrUuRttnMbln6tX+A6F9+0dIDwvMDJkWt4Rlwt2ZyDsklloN21r8tahndMIBq2OChq7thBu5sbRnzNhNvCtmpqrQLCTvmiLAtlxvXtqCC7bl2wQILovtoBpUNWFshsWtptNt9yCNqpsAwHBIv2n/AdmdpktyJtFMHBrANk9xRtWRsOMLpmQtt1qhNUBvqsO5lstDVqktmluDsfdq2NvNrVtw5sPsUFkrNWRyXumxscsRhiotmVoBtmVsgsEFmHtmtspNwBsSsGphKtDptKOctuOsV9mht69tFNpFqIMexlUtbNrONLJx6Mq5ldti5tQNAxqYMvhlAte5rCtczwAACtTYy///+pYxJvAAAK1Mc9wJtrlohuG5hLwHN6v2s9AdtpuNQXXtu2CTBdR1vgtkJtutEVhQsn5t+NV5jutQliKNGVwftxdsANuhgKtExtlNDlp7N1RhOMstuCNetr9uR6A4p1+ynIDs4NI1ubOB9gmMH9rttN9qGNwRgWM3pqOtdZo/OWlhvNI5stNhNr7txZjRsvtqgN1ZpcNUBjhseRwKt7lytNzBgStANl2NQBu0NLxh6NQJtMtmlqkuEhjstlBsDuHZqkNixhEs7lqnNppt5NuNiXMPxt4NyNvDNb9hyNU9rztxhsKNUNhAsi9rMtuNvINqdgMMM1lFuM5u5NlVgUs+BpAtydqyNstk4MlppstXZtTt8hhSs0NrBtcxqWNQtiOQHV3v2ihAdVRpONw1lLtZhrJNzRrLtntjRMlJrGNc5qCtF1izMtZtAtOdtoNhJhEMrRrxtFVv0QXW4u2MnBczBsfuYtrrtLmC4oN21nYLoqQAA==", // nolint:lll + "FiYqRnIdAACAQEAAAAAPGEnQDAfgAATiCSSSSSSSSSSSSSSSSSSSSSSSfAAAnEH///2PBJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJMBb8wFJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJgAA==", // nolint:lll + "FiYqRnIdAACAQEAAAAAarE7gA+AABOIJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJMAAA==", // nolint:lll + "FiYqRnIdAACAQEAAAAA94IfIDfYY0GwPwAAJxDAAH592AASFBMAAeXVYAA/OkGiuzWt4Dtu8tNrDNvACt9dFxQKt9lRx22OG9JwIlN+YFwUkN241uCOOBi1tzUN8F1u9zOC6pv9fNrUVvQot5dNv2vtzelurVuBihvh2t6cdv+Pt3lRwKeOKG1uHjN/bCA79Rd+3xD4DvJgtxP9vwEt8rlut/t5p9vjCOLHdwTTN0k1u2QN5j1wVWN5/RxHnt0OVuQkN7KFwsetm9xxfeN4pBuxpOBGtuE8NzABwc2OHS1wc9OAtZvmPt6N1wXJNtN5vjWt+DVsstt45Bv98t2w5xRTN2UxvLnt5/hvD3ODCJvWluAQBxadOBxhv/AOAGVxGgODJpvXcOB3VwapN7xJxN4uH7pvK5OI1Rt2/uAoiB704V631N4HwsjN6gJwmkuMEZwKZN7G9ujTNz7lwnkN6AtvLwN4Xhvv5t7ZJw6UtvS5w8xN/O5u6Stxi5wU5N25VwkaNy4xu3cOAEZwK6NvnBvWZN8KFuBXN6mFwvqN9/Zw5EOEZ5vJBN5yBw9IuTW5ueOt9BRv/aOCLVvxst99Jv6dtvNNu5juA5VwxVN8lJuYtt56tvwlt+BJw9vt79dwAqN5PhuyMNnopuzcN8WpvbKN6MZuaBNx09yyzt1blxItt6lpw50OEGtxZSN2LBwvat8BVvv+tyM5xGxt5xNwjSuGR1wV8OEuxv0Stz1Bx30ODlNvu0t14pvqzN66Rv85t2WFvhzuBBBvJgN6G9vlBNz4hv06N77NvFOt2/hvWZuD1ZuzmNu25vqEOR1Vu1StzdFv4SuKDluEON4mlu6juC4JvPTORxVvgdN8G1v7fN6epvTsN7pltzKuOeRuwyODvVw0tOB71vxGt+hVvYvt8plxfzN+UNwBoOJgtxZCt9R5woct6gpxBNt9U9wRdt4P1vkeNsP1wmzN43VvlYNzbtuwxt0jJuqQOG0hvzwN27Rv2uuGQNuaHOIeVvWVOEtBtgHN8NtybruAM9ws9N/s5w2ot711wEzN5n9w6ZN84JvaetoexwmhOI5puccNv1VuLaOBlRxAft04VupXN6d9v9ktvchv87tswVvieN4EBvVvuJ99vTvOHhlv1SOH81u7jNwFtuABN1lFxUZN0GNxMzN5PtwGmN9T9wE9t9plup6t/QRwV6OOltt5PN3Iluszt+elwZlt95BwmOuK1tvzaOC1duCANyk5wHfuD5RvT9wTeFLvG8BtBOCqRwZ4uKQBwszt/ZBv/pt7AZvDFtq/xtwSN9PaA7jy1+4nOIDuXWt3fJvnwNsoRvILNsG1v8UOBWFvlct9utwqMOKOxujKN4tZvmAtwhhvjgOMWhvQoN1xRu/Ct629uXtt3EhwAhN7CBvgwNwPhwpWttsBwexNwj1xGPN9sRu00N+Ydu4Zt/OtwFht9rxwzktwZ1w8vNyHZwzLtwLVuseN6hRu69t/ptvMct5tZuv/uEstwa8t/lluvTOEmpws3N9TRumCuAUhtc8NzvlvhCt7ftvYvt4YBurmt2gJv3jN1idv4eN6cxve9N2MVuliuPUpvg5t8V5vjxtuWJvE0t9ZFvKQOB91wsYt9CqA73Gjcd3vu7tvAt1SRwoHtu19xTbOTKdtsPuKkBvdft6ChxqHN8WdugRNy9VvEbOErBwBVt7kFwNpuHixvF9t8xNvjyt2KtuUltondvSftzpttyLt3zNu8nN0QZvEhN4WZxNrt2Jtt2NN3AtvoNNyUlvWouN1Fw+iN/ZBvs8tl65xiDtz1Zwm0NyxduGSOBsJua9t4otvEjt1G1vYVNyQduiFOCxdw+KuGLtuyrN6s9wTztyHhvy0uKfJueHuMZpvIHN0pZxU1N7KhuliuBtxwFgt5PJty0QLeb2vm8RtAt615v8TN2IdwJ4tzWBtHFuIYdvYwuATlx6ptxC1wD7t8D1v0RN9w5utAt8d9wmJN4kVxPrOGGVvM1t/BVvRON3cZyJiN3ltv5QN/FtvLLtrYBw3ZuHj5wiwt/v1usHN5e1wu+N3Wdw1qN9D1w8Yt52VwkQt4iZxJctvfVvvqOLyhxIBNyPRujWuD6htvQt9StvI+t/utw2Vt8ppxLAttYpvjeN/ShvpXNvChvfst7cVvuTuHlRt/aN53hv6JN44RwyYynfEh2u3sVZTvzIuAd5u7CON7BvBiNyMJvoGOQHBuneuDzltU1t4hVwDqN2yBwJ8N88Bu49t1mdvNfOBbZwS3uPfJwaNNzzdwPIN63xukQN+ItviCwHesev21zxAeDhRwiLtqQJuoFt7cFw2QtsBhwYMuPyVuklNtDVwfwt8P5wZxNzNpu+RtkcVwBLNp+Vvmzt/8Rwo/OIq5uKcNtLJuQxuBhxvMLN/r5wl1Nyzpts7tseZvmDuGLRt5LN8FhwlmORGxvqtOBPRvkIt6FBu64t8MdwSbt8S1woEt0Ctt7otuI9wyut0GRwNiOFppuoxuBAFwJAOKkZuNrOBqtt+KOA3NxaYN5WdtXxOCzdwrNN4uBwyzN9qlwjkN5CBwBHOQJFtnlOD6pw5tt2K9xl3ONI1wx+uC8mAA", // nolint:lll + "FiYqRnIdAACAQEAAAABGId9oDfcTD+KTwAAJxDAAJ9nmAAgRqMAAyOLYABNsmGkqKtGlSxai45t0OWyDGakfNqCDmpc7a1FhtxUW0JAagCBq4kWiuPaWIVuiqWx1DaZ8dpS42tOuau7NtH526t8adBRqMy2iVZabJtuBPoDuMH79pdk4DqL6Gn02aggVucnYFt0QL7qE0Wme4gOlusabFdtjBmuwRado5pei4Dp9lr9qEkIDuFg2yO4aVwppYX2oPOaYWFtkKm2OWaT3BpbpmnylanWZuI5m2XBab15puEWrJ4aSDBtt7W8VuayVFqRSGo68aX9RvhI22xIaNUVqXlmneXaOlpuUQGsIqaWoNqHvGlF1afUFtA8mwQ3aoENp6pWqtIay+1ssXG1j2aiLFqnZ2qTXaYa5uSFm9nYaW/tqnLmsU0ay+Rt2sG+vCaK2dqG4movralhdt+vm6W+aro9psSGi3/aXIZv2A21//aitVqG0YJps07xp5hIJtyDGq4lahdZrPB2omPabEhs3HG0nQacY9re2IDo9xb9qOc4DtrTm1Q3aNUVqrHWocCabaxtp7WtdEaYflq4FYDpn5L9pt04DuSY22gTawH9pMP2ng1gOqgMv25S4gO1kgaiLdpf62l0xaaTtu0ZG2Jwal7RpDT2lwaajbWA7kwq/bk5OA6Y8xqtWmnWOaaYls0J2xFVgOrKov2l0bgOlylaUGVvBlm5XIaiDlqyj2mHyareRuR92we7aPPJq44Wm0pakR9sznG6g3amm5qZ+2rCuarOVvZnG4NoaP19rOYGoXpameRt1bG05SafENpkjWmimafb5uFwW0Acagj9qIY2kdhap3tvQWmwEKadOFouV2sIraX+1tEcWwZ7a6+dqePmkkfgOk4dv24fIgWpYDv2nBvgOqscabjlqFd23mfgOylbv2oj2gWokrv2l5OgOpn4cAMxsNxWquhamiNqY+Gr6ubqCRsSsmpuvakdpp7aYDqnmr9vStYDrdPmqk6gOozNv2qYLgOsQxbwpNvfsGoc6alFFpT+mpnubhB1tEE2lTSav/hoYVWky0b72tuuammghak85qbDGqJBbbN9uOgoDqZW79qbJYDqg0mjIwbmVJtKpWrszgOpmuv2oi4gOnmbbYyFtYa2o6Yawi5pjvGojnbbG5ub/2pycafIxpZY2mJBgO/Szv2vr6gOmuzgOk8tv2nmBgOoEibcKdsfzmkSHahcVqN5Wicub5jxuoi2pqIgOrbpv2oM/gOn7abgLtuBN2nunazp1o0/GtrEbcRZt/0oDqkib9pHMIDqHxGmRvbsCtq88GrROaN2SA6Y0m/aVMeA7tBxuQvWoKMas3hrPTmojKbqsRq202lpxartdqZKYDqjr79s/7YDvOymnglao+5q5qmrC/g23crumwa4g2qgqaehtobo2mKEbwzlsWsGrlRaReNpOkGoYmbMD1vYDGi0zafyNq+32j5ybJT9uW9mrLSaVmlqAbWqzIbic9q1BWqDIahZJpWZ2nNhbkOttxMWjWKaf8hpnAWkA1bcd5stqmjjXal3ppTR4DrJeL9tieYDtL82o3YacAZq3B2plQbgM5thJGljMakTZorqmqSnbiFpr2qWn5ran9pp/qGk9gbgH9sd1mojAawPFpzL2nKDbYfhuYuGqdRaRa5qihGiwWbg21uRDmmFWajuZpnMGmipbmjBt222k90abMVqieWktEbB59vTxWq69aq4xqX22mAdbeXKA60BG/afuaA6TK5qNxGtW/b58dr+B2le2adQhpi1mnpEbLaptC+WxijaTpdqrRGrHbbpUBtuY2p4QaYuKCaXBq8aexeCboedqfCGn2Maq+KCaxO28abA6CbbRhty0mnV5aOa1oqvIDqvsr9sPNoDs532m+QaX7hpa/moolbaBRwqB2kGDaXnBpKHGqRQa+Z5u5UGqqba6w5pycGow2bHhxtVV2lTsaq4Vp3BWkh/bQNNta2WoGgacIVp1DGnfGbPZht5PWsiyaoWRqEwGmdlgO4Aov21sQgOmNBamh5rJv2jKvbtyps7AmqsoaphRqA7WnIlbVyRu4kWmb5aMNJpiJ2oyKa6F9uNi2mfDakyaA6Z7+/aX9qA7VVRsvkmrJuaqwmA6Lda/ayU2A7Xp9r+22oKdal5dqY24Dp0tb9uUtYDtSommY3achJpnNmjiJbYyNusUWlqpamUBqHYWo5/bW79s27mrZmahtFpqEmm8ubhK5ujlGnGHanJdpdYWsXXbza5vQ2mnqgamiFpxZGoLRb5/9uApmmaFacUBpiVmmmbb3dhrWRmnBAapKVpudGre6gOyNuv2uhPgOo3KahcBpgmGndWbcXluzgGpq7abVlp2poDqcFr9tdVYDuIbGmtqakJ9qT02pfvbg/dthuGkrPaV7FqCM2sNHbX9Vt1r4Douu79qKjYDqaPmrwybtK1se0GooVaSj9qDnmrVUb0/eA7AL6/asHmBaSge/6jS6/6hceA7RxdtDQmoK5aefRo/eGqwVborls13mk0/ak8qA6vRW/aS6GA7Z55ukIWmsWaLzlqve2rXJbRSVtZ62modaGH9qWs2jQAbzFxvBWWtgsalnVo/72iaEbSbOBbPYu+agDOBacCpqSLoJporLxsXrIJs8HWuUoaZPtpHzWlefbdO5u58mm7jat7Rpj5Gne3beCdorCmn3OafItqKYmmTTbHw9s2k2q4ZaeXBrIF4FqYcr5t8q4FuKkmm4Aa2Cpol42q7fbsKdtK2GtYGae19ouOWsWEbWTFuNfmpClaiXZqCnWrYgbWVJu9n2kt6apzFrDcmmGAb2lhsT3IDqR3b9qSSYAA=", // nolint:lll +} + +func BenchmarkM3TSZEncode(b *testing.B) { + var ( + encodingOpts = encoding.NewOptions() + seriesRun = prepareSampleSeriesEncRun(b) + encoder = NewEncoder(time.Now(), nil, DefaultIntOptimizationEnabled, encodingOpts) + ) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + run := seriesRun[i] + encoder.Reset(run[0].Timestamp, len(run), nil) + + for i := range run { + // Using index access to avoid copying a 40 byte datapoint. + _ = encoder.Encode(run[i], xtime.Nanosecond, nil) + } + + encoder.Discard() + } +} + +func prepareSampleSeriesEncRun(b *testing.B) [][]ts.Datapoint { + var ( + rnd = rand.New(rand.NewSource(42)) // nolint:gosec + sampleSeries = make([][]byte, 0, len(sampleSeriesBase64)) + seriesRun = make([][]ts.Datapoint, b.N) + encodingOpts = encoding.NewOptions() + reader = bytes.NewReader(nil) + ) + + for _, b64 := range sampleSeriesBase64 { + data, err := base64.StdEncoding.DecodeString(b64) + require.NoError(b, err) + + sampleSeries = append(sampleSeries, data) + } + + for i := 0; i < len(seriesRun); i++ { + reader.Reset(sampleSeries[rnd.Intn(len(sampleSeries))]) + + iter := NewReaderIterator(reader, DefaultIntOptimizationEnabled, encodingOpts) + for iter.Next() { + dp, _, _ := iter.Current() + seriesRun[i] = append(seriesRun[i], dp) + } + + require.NoError(b, iter.Err()) + iter.Close() + } + + return seriesRun +} From 6536b7cd68d3ba2c4f84a113dd6da42bed8293e0 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Mon, 9 Nov 2020 12:58:38 -0500 Subject: [PATCH 39/47] [etcd] Set reasonable cluster connection/sync settings by default (#2860) --- src/cluster/client/etcd/client.go | 6 +++-- src/cluster/client/etcd/config_test.go | 6 ++--- src/cluster/client/etcd/options.go | 28 +++++++++++++++---- src/cluster/client/etcd/options_test.go | 36 ++++++++++++++++++++----- src/cluster/client/etcd/types.go | 5 +++- 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/cluster/client/etcd/client.go b/src/cluster/client/etcd/client.go index c3218c2edf..88f85e8dee 100644 --- a/src/cluster/client/etcd/client.go +++ b/src/cluster/client/etcd/client.go @@ -39,8 +39,8 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/retry" - "go.etcd.io/etcd/clientv3" "github.com/uber-go/tally" + "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) @@ -282,9 +282,10 @@ func newClient(cluster Cluster) (*clientv3.Client, error) { return nil, err } cfg := clientv3.Config{ + AutoSyncInterval: cluster.AutoSyncInterval(), + DialTimeout: cluster.DialTimeout(), Endpoints: cluster.Endpoints(), TLS: tls, - AutoSyncInterval: cluster.AutoSyncInterval(), } if opts := cluster.KeepAliveOptions(); opts.KeepAliveEnabled() { @@ -296,6 +297,7 @@ func newClient(cluster Cluster) (*clientv3.Client, error) { } cfg.DialKeepAliveTime = keepAlivePeriod cfg.DialKeepAliveTimeout = opts.KeepAliveTimeout() + cfg.PermitWithoutStream = true } return clientv3.New(cfg) diff --git a/src/cluster/client/etcd/config_test.go b/src/cluster/client/etcd/config_test.go index 62c6af66eb..f2c5bb2f5a 100644 --- a/src/cluster/client/etcd/config_test.go +++ b/src/cluster/client/etcd/config_test.go @@ -138,9 +138,9 @@ m3sd: require.True(t, exists) keepAliveOpts = cluster2.KeepAliveOptions() require.Equal(t, true, keepAliveOpts.KeepAliveEnabled()) - require.Equal(t, 5*time.Minute, keepAliveOpts.KeepAlivePeriod()) - require.Equal(t, 5*time.Minute, keepAliveOpts.KeepAlivePeriodMaxJitter()) - require.Equal(t, 20*time.Second, keepAliveOpts.KeepAliveTimeout()) + require.Equal(t, 20*time.Second, keepAliveOpts.KeepAlivePeriod()) + require.Equal(t, 10*time.Second, keepAliveOpts.KeepAlivePeriodMaxJitter()) + require.Equal(t, 10*time.Second, keepAliveOpts.KeepAliveTimeout()) t.Run("TestOptionsNewDirectoryMode", func(t *testing.T) { opts := cfg.NewOptions() diff --git a/src/cluster/client/etcd/options.go b/src/cluster/client/etcd/options.go index e6e3c719de..566d240e10 100644 --- a/src/cluster/client/etcd/options.go +++ b/src/cluster/client/etcd/options.go @@ -36,10 +36,13 @@ import ( ) const ( + defaultAutoSyncInterval = 1 * time.Minute + defaultDialTimeout = 15 * time.Second + defaultKeepAliveEnabled = true - defaultKeepAlivePeriod = 5 * time.Minute - defaultKeepAlivePeriodMaxJitter = 5 * time.Minute - defaultKeepAliveTimeout = 20 * time.Second + defaultKeepAlivePeriod = 20 * time.Second + defaultKeepAlivePeriodMaxJitter = 10 * time.Second + defaultKeepAliveTimeout = 10 * time.Second defaultRetryInitialBackoff = 2 * time.Second defaultRetryBackoffFactor = 2.0 @@ -316,8 +319,10 @@ func (o options) NewDirectoryMode() os.FileMode { // NewCluster creates a Cluster. func NewCluster() Cluster { return cluster{ - keepAliveOpts: NewKeepAliveOptions(), - tlsOpts: NewTLSOptions(), + autoSyncInterval: defaultAutoSyncInterval, + dialTimeout: defaultDialTimeout, + keepAliveOpts: NewKeepAliveOptions(), + tlsOpts: NewTLSOptions(), } } @@ -327,6 +332,7 @@ type cluster struct { keepAliveOpts KeepAliveOptions tlsOpts TLSOptions autoSyncInterval time.Duration + dialTimeout time.Duration } func (c cluster) Zone() string { @@ -373,3 +379,15 @@ func (c cluster) SetAutoSyncInterval(autoSyncInterval time.Duration) Cluster { c.autoSyncInterval = autoSyncInterval return c } + +//nolint:gocritic +func (c cluster) DialTimeout() time.Duration { + return c.dialTimeout +} + +//nolint:gocritic +func (c cluster) SetDialTimeout(dialTimeout time.Duration) Cluster { + c.dialTimeout = dialTimeout + + return c +} diff --git a/src/cluster/client/etcd/options_test.go b/src/cluster/client/etcd/options_test.go index aee7366fab..befd638960 100644 --- a/src/cluster/client/etcd/options_test.go +++ b/src/cluster/client/etcd/options_test.go @@ -32,16 +32,22 @@ import ( ) func TestKeepAliveOptions(t *testing.T) { - opts := NewKeepAliveOptions(). + opts := NewKeepAliveOptions() + require.Equal(t, defaultKeepAliveEnabled, opts.KeepAliveEnabled()) + require.Equal(t, defaultKeepAlivePeriod, opts.KeepAlivePeriod()) + require.Equal(t, defaultKeepAlivePeriodMaxJitter, opts.KeepAlivePeriodMaxJitter()) + require.Equal(t, defaultKeepAliveTimeout, opts.KeepAliveTimeout()) + + opts = NewKeepAliveOptions(). SetKeepAliveEnabled(true). - SetKeepAlivePeriod(10 * time.Second). - SetKeepAlivePeriodMaxJitter(5 * time.Second). - SetKeepAliveTimeout(time.Second) + SetKeepAlivePeriod(1234 * time.Second). + SetKeepAlivePeriodMaxJitter(5000 * time.Second). + SetKeepAliveTimeout(time.Hour) require.Equal(t, true, opts.KeepAliveEnabled()) - require.Equal(t, 10*time.Second, opts.KeepAlivePeriod()) - require.Equal(t, 5*time.Second, opts.KeepAlivePeriodMaxJitter()) - require.Equal(t, time.Second, opts.KeepAliveTimeout()) + require.Equal(t, 1234*time.Second, opts.KeepAlivePeriod()) + require.Equal(t, 5000*time.Second, opts.KeepAlivePeriodMaxJitter()) + require.Equal(t, time.Hour, opts.KeepAliveTimeout()) } func TestCluster(t *testing.T) { @@ -63,6 +69,22 @@ func TestCluster(t *testing.T) { assert.Equal(t, "z", c.Zone()) assert.Equal(t, []string{"e1"}, c.Endpoints()) assert.Equal(t, aOpts, c.TLSOptions()) + assert.Equal(t, defaultAutoSyncInterval, c.AutoSyncInterval()) + assert.Equal(t, defaultDialTimeout, c.DialTimeout()) + + c = c.SetAutoSyncInterval(123 * time.Minute) + assert.Equal(t, "z", c.Zone()) + assert.Equal(t, []string{"e1"}, c.Endpoints()) + assert.Equal(t, aOpts, c.TLSOptions()) + assert.Equal(t, 123*time.Minute, c.AutoSyncInterval()) + assert.Equal(t, defaultDialTimeout, c.DialTimeout()) + + c = c.SetDialTimeout(42 * time.Hour) + assert.Equal(t, "z", c.Zone()) + assert.Equal(t, []string{"e1"}, c.Endpoints()) + assert.Equal(t, aOpts, c.TLSOptions()) + assert.Equal(t, 123*time.Minute, c.AutoSyncInterval()) + assert.Equal(t, 42*time.Hour, c.DialTimeout()) } func TestTLSOptions(t *testing.T) { diff --git a/src/cluster/client/etcd/types.go b/src/cluster/client/etcd/types.go index ffdc284bac..9c052d1e4f 100644 --- a/src/cluster/client/etcd/types.go +++ b/src/cluster/client/etcd/types.go @@ -129,6 +129,9 @@ type Cluster interface { TLSOptions() TLSOptions SetTLSOptions(TLSOptions) Cluster - SetAutoSyncInterval(value time.Duration) Cluster AutoSyncInterval() time.Duration + SetAutoSyncInterval(value time.Duration) Cluster + + DialTimeout() time.Duration + SetDialTimeout(value time.Duration) Cluster } From ce720e8ab0c532421a2bac3ff724edbdfadb447d Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Mon, 9 Nov 2020 13:01:12 -0500 Subject: [PATCH 40/47] [m3cluster] Expose placement algorithm in placement service (#2858) --- .../service/mirrored_custom_groups_test.go | 5 +- src/cluster/placement/service/operator.go | 5 +- .../placement/service/operator_test.go | 15 ++-- src/cluster/placement/service/service.go | 66 +++++++++++++--- src/cluster/placement/service/service_test.go | 77 +++++++++++++------ src/cluster/services/services.go | 2 +- .../m3coordinator/downsample/options.go | 2 +- src/msg/integration/setup.go | 5 +- .../writer/consumer_service_writer_test.go | 7 +- .../api/v1/handler/placement/get_test.go | 4 +- 10 files changed, 134 insertions(+), 54 deletions(-) diff --git a/src/cluster/placement/service/mirrored_custom_groups_test.go b/src/cluster/placement/service/mirrored_custom_groups_test.go index a27c9df514..1d624aaaef 100644 --- a/src/cluster/placement/service/mirrored_custom_groups_test.go +++ b/src/cluster/placement/service/mirrored_custom_groups_test.go @@ -59,7 +59,6 @@ const ( instG3I1 = "g3_i1" instG3I2 = "g3_i2" instG3I3 = "g3_i3" - ) var ( @@ -205,7 +204,7 @@ func mirroredCustomGroupSelectorSetup(t *testing.T) *mirroredCustomGroupSelector tctx.Groups = testGroups opts := placement.NewOptions(). - SetValidZone(zone). + SetValidZone(zone). SetIsMirrored(true) tctx.Selector = selector.NewMirroredCustomGroupSelector( @@ -217,7 +216,7 @@ func mirroredCustomGroupSelectorSetup(t *testing.T) *mirroredCustomGroupSelector tctx.KVStore = mem.NewStore() tctx.Storage = placementstorage.NewPlacementStorage(tctx.KVStore, "placement", tctx.Opts) - tctx.Service = NewPlacementService(tctx.Storage, tctx.Opts) + tctx.Service = NewPlacementService(tctx.Storage, WithPlacementOptions(tctx.Opts)) return tctx } diff --git a/src/cluster/placement/service/operator.go b/src/cluster/placement/service/operator.go index 75da980fc4..5d253969cc 100644 --- a/src/cluster/placement/service/operator.go +++ b/src/cluster/placement/service/operator.go @@ -31,10 +31,10 @@ import ( // given placement. // If initialPlacement is nil, BuildInitialPlacement must be called before any operations on the // placement. -func NewPlacementOperator(initialPlacement placement.Placement, opts placement.Options) placement.Operator { +func NewPlacementOperator(initialPlacement placement.Placement, opts ...Option) placement.Operator { store := newDummyStore(initialPlacement) return &placementOperator{ - placementServiceImpl: newPlacementServiceImpl(opts, store), + placementServiceImpl: newPlacementServiceImpl(store, opts...), store: store, } } @@ -97,4 +97,3 @@ func (d *dummyStore) Placement() (placement.Placement, error) { } return d.curPlacement, nil } - diff --git a/src/cluster/placement/service/operator_test.go b/src/cluster/placement/service/operator_test.go index af3b1b8688..3fcf1132e0 100644 --- a/src/cluster/placement/service/operator_test.go +++ b/src/cluster/placement/service/operator_test.go @@ -25,7 +25,6 @@ import ( "testing" "github.com/m3db/m3/src/cluster/placement" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,13 +32,13 @@ import ( func TestOperator(t *testing.T) { type testDeps struct { options placement.Options - op placement.Operator + op placement.Operator } setup := func(t *testing.T) testDeps { options := placement.NewOptions().SetAllowAllZones(true) return testDeps{ options: options, - op: NewPlacementOperator(nil, options), + op: NewPlacementOperator(nil, WithPlacementOptions(options)), } } @@ -60,7 +59,7 @@ func TestOperator(t *testing.T) { t.Run("end-to-end flow", func(t *testing.T) { tdeps := setup(t) - op := NewPlacementOperator(nil, tdeps.options) + op := NewPlacementOperator(nil, WithPlacementOptions(tdeps.options)) store := newMockStorage() pl, err := op.BuildInitialPlacement([]placement.Instance{newTestInstance()}, 10, 1) @@ -81,7 +80,7 @@ func TestOperator(t *testing.T) { require.NoError(t, err) // expect exactly one version increment, from store.SetIfNotExist - assert.Equal(t, initialVersion + 1, pl.Version()) + assert.Equal(t, initialVersion+1, pl.Version()) // spot check the results allAvailable := true @@ -94,15 +93,15 @@ func TestOperator(t *testing.T) { }) } -type dummyStoreTestDeps struct{ +type dummyStoreTestDeps struct { store *dummyStore - pl placement.Placement + pl placement.Placement } func dummyStoreSetup(t *testing.T) dummyStoreTestDeps { return dummyStoreTestDeps{ store: newDummyStore(nil), - pl: placement.NewPlacement(), + pl: placement.NewPlacement(), } } diff --git a/src/cluster/placement/service/service.go b/src/cluster/placement/service/service.go index 67873db30f..2ee4331c9c 100644 --- a/src/cluster/placement/service/service.go +++ b/src/cluster/placement/service/service.go @@ -27,7 +27,6 @@ import ( "github.com/m3db/m3/src/cluster/placement/algo" "github.com/m3db/m3/src/cluster/placement/selector" "github.com/m3db/m3/src/cluster/shard" - "go.uber.org/zap" ) @@ -37,36 +36,79 @@ type placementService struct { } // NewPlacementService returns an instance of placement service. -func NewPlacementService(s placement.Storage, opts placement.Options) placement.Service { +func NewPlacementService(s placement.Storage, opts ...Option) placement.Service { return &placementService{ Storage: s, placementServiceImpl: newPlacementServiceImpl( - opts, s, - + opts..., ), } } +type options struct { + placementAlgorithm placement.Algorithm + placementOpts placement.Options +} + +// Option is an interface for PlacementService options. +type Option interface { + apply(*options) +} + +// WithAlgorithm sets the algorithm implementation that will be used by PlacementService. +func WithAlgorithm(algo placement.Algorithm) Option { + return &algorithmOption{placementAlgorithm: algo} +} + +type algorithmOption struct { + placementAlgorithm placement.Algorithm +} + +func (a *algorithmOption) apply(opts *options) { + opts.placementAlgorithm = a.placementAlgorithm +} + +type placementOptionsOption struct { + opts placement.Options +} + +func (a *placementOptionsOption) apply(opts *options) { + opts.placementOpts = a.opts +} + +// WithPlacementOptions sets the placement options for PlacementService. +func WithPlacementOptions(opts placement.Options) Option { + return &placementOptionsOption{opts: opts} +} + func newPlacementServiceImpl( - opts placement.Options, storage minimalPlacementStorage, + opts ...Option, ) *placementServiceImpl { - if opts == nil { - opts = placement.NewOptions() + o := options{ + placementOpts: placement.NewOptions(), + } + + for _, opt := range opts { + opt.apply(&o) + } + + if o.placementAlgorithm == nil { + o.placementAlgorithm = algo.NewAlgorithm(o.placementOpts) } - instanceSelector := opts.InstanceSelector() + instanceSelector := o.placementOpts.InstanceSelector() if instanceSelector == nil { - instanceSelector = selector.NewInstanceSelector(opts) + instanceSelector = selector.NewInstanceSelector(o.placementOpts) } return &placementServiceImpl{ store: storage, - opts: opts, - algo: algo.NewAlgorithm(opts), + opts: o.placementOpts, + algo: o.placementAlgorithm, selector: instanceSelector, - logger: opts.InstrumentOptions().Logger(), + logger: o.placementOpts.InstrumentOptions().Logger(), } } diff --git a/src/cluster/placement/service/service_test.go b/src/cluster/placement/service/service_test.go index 5ebe2b49db..344454df28 100644 --- a/src/cluster/placement/service/service_test.go +++ b/src/cluster/placement/service/service_test.go @@ -26,15 +26,16 @@ import ( "github.com/m3db/m3/src/cluster/kv/mem" "github.com/m3db/m3/src/cluster/placement" + "github.com/m3db/m3/src/cluster/placement/algo" "github.com/m3db/m3/src/cluster/placement/storage" "github.com/m3db/m3/src/cluster/shard" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestGoodWorkflow(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) testGoodWorkflow(t, p) } @@ -145,7 +146,8 @@ func assertPlacementInstanceEqualExceptShards( } func TestNonShardedWorkflow(t *testing.T) { - ps := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1").SetIsSharded(false)) + ps := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1").SetIsSharded(false))) _, err := ps.BuildInitialPlacement([]placement.Instance{ placement.NewEmptyInstance("i1", "r1", "z1", "e1", 1), @@ -206,7 +208,8 @@ func TestNonShardedWorkflow(t *testing.T) { } func TestBadInitialPlacement(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1").SetIsSharded(false)) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1").SetIsSharded(false))) // invalid numShards _, err := p.BuildInitialPlacement([]placement.Instance{ @@ -229,7 +232,8 @@ func TestBadInitialPlacement(t *testing.T) { }, 10, 1) assert.Error(t, err) - p = NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) // Not enough instances. _, err = p.BuildInitialPlacement([]placement.Instance{}, 10, 1) @@ -264,7 +268,8 @@ func TestBadInitialPlacement(t *testing.T) { } func TestBadAddReplica(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err := p.BuildInitialPlacement( []placement.Instance{placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 1)}, @@ -276,14 +281,16 @@ func TestBadAddReplica(t *testing.T) { assert.Error(t, err) // Could not find placement for service. - p = NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err = p.AddReplica() assert.Error(t, err) } func TestBadAddInstance(t *testing.T) { ms := newMockStorage() - p := NewPlacementService(ms, placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(ms, + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err := p.BuildInitialPlacement( []placement.Instance{placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 1)}, @@ -298,18 +305,21 @@ func TestBadAddInstance(t *testing.T) { _, _, err = p.AddInstances([]placement.Instance{placement.NewEmptyInstance("i2", "r2", "z2", "endpoint", 1)}) assert.Error(t, err) - p = NewPlacementService(ms, placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(ms, + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, _, err = p.AddInstances([]placement.Instance{placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 1)}) assert.Error(t, err) // could not find placement for service - p = NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, _, err = p.AddInstances([]placement.Instance{placement.NewEmptyInstance("i2", "r2", "z1", "endpoint", 1)}) assert.Error(t, err) } func TestBadRemoveInstance(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err := p.BuildInitialPlacement( []placement.Instance{placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 1)}, @@ -325,13 +335,15 @@ func TestBadRemoveInstance(t *testing.T) { assert.Error(t, err) // Could not find placement for service. - p = NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err = p.RemoveInstances([]string{"i1"}) assert.Error(t, err) } func TestBadReplaceInstance(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err := p.BuildInitialPlacement([]placement.Instance{ placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 1), @@ -363,7 +375,8 @@ func TestBadReplaceInstance(t *testing.T) { assert.Error(t, err) // Could not find placement for service. - p = NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p = NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, _, err = p.ReplaceInstances( []string{"i1"}, []placement.Instance{placement.NewEmptyInstance("i2", "r2", "z1", "endpoint", 1)}, @@ -406,7 +419,8 @@ func TestMarkShard(t *testing.T) { _, err := ms.SetIfNotExist(p) assert.NoError(t, err) - ps := NewPlacementService(ms, placement.NewOptions().SetValidZone("z1")) + ps := NewPlacementService(ms, + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) _, err = ps.MarkShardsAvailable("i5", 1) assert.NoError(t, err) p, err = ms.Placement() @@ -461,7 +475,7 @@ func TestMarkInstance(t *testing.T) { _, err := ms.SetIfNotExist(p) assert.NoError(t, err) - ps := NewPlacementService(ms, placement.NewOptions().SetValidZone("z1")) + ps := NewPlacementService(ms, WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) // instance not exist _, err = ps.MarkInstanceAvailable("i6") @@ -553,7 +567,7 @@ func TestFindReplaceInstance(t *testing.T) { }, } for _, test := range testCases { - p := NewPlacementService(nil, test.opts).(*placementService) + p := NewPlacementService(nil, WithPlacementOptions(test.opts)).(*placementService) res, err := p.selector.SelectReplaceInstances(test.input, test.replaceIDs, s) if test.expectErr { assert.Error(t, err) @@ -663,7 +677,7 @@ func TestMirrorWorkflow(t *testing.T) { ps := NewPlacementService( newMockStorage(), - placement.NewOptions().SetValidZone("z1").SetIsMirrored(true), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1").SetIsMirrored(true)), ) p, err := ps.BuildInitialPlacement( @@ -758,7 +772,8 @@ func TestMirrorWorkflow(t *testing.T) { } func TestManyShards(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))) i1 := placement.NewEmptyInstance("i1", "r1", "z1", "endpoint", 2) i2 := placement.NewEmptyInstance("i2", "r2", "z1", "endpoint", 2) i3 := placement.NewEmptyInstance("i3", "r3", "z1", "endpoint", 2) @@ -816,7 +831,7 @@ func TestAddMultipleInstances(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ps := NewPlacementService(newMockStorage(), test.opts) + ps := NewPlacementService(newMockStorage(), WithPlacementOptions(test.opts)) _, err := ps.BuildInitialPlacement(test.initialInstances, 4, 2) require.NoError(t, err) @@ -897,7 +912,7 @@ func TestReplaceInstances(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ps := NewPlacementService(newMockStorage(), test.opts) + ps := NewPlacementService(newMockStorage(), WithPlacementOptions(test.opts)) _, err := ps.BuildInitialPlacement(test.initialInstances, 4, 2) require.NoError(t, err) @@ -913,7 +928,8 @@ func TestReplaceInstances(t *testing.T) { } func TestValidateFnBeforeUpdate(t *testing.T) { - p := NewPlacementService(newMockStorage(), placement.NewOptions().SetValidZone("z1")).(*placementService) + p := NewPlacementService(newMockStorage(), + WithPlacementOptions(placement.NewOptions().SetValidZone("z1"))).(*placementService) _, err := p.BuildInitialPlacement( []placement.Instance{placement.NewEmptyInstance("i1", "r1", "z1", "endpoint1", 1)}, @@ -927,6 +943,23 @@ func TestValidateFnBeforeUpdate(t *testing.T) { assert.Equal(t, expectErr, err) } +func TestPlacementServiceImplOptions(t *testing.T) { + placementOptions := placement.NewOptions().SetValidZone("foozone").SetIsSharded(true) + al := algo.NewAlgorithm(placementOptions.SetIsSharded(false)) + + defaultImpl := newPlacementServiceImpl(nil) + require.NotNil(t, defaultImpl) + assert.NotNil(t, defaultImpl.opts) + assert.NotNil(t, defaultImpl.algo) + assert.NotEqual(t, placementOptions.ValidZone(), defaultImpl.opts.ValidZone()) + + customImpl := newPlacementServiceImpl(nil, + WithPlacementOptions(placementOptions), + WithAlgorithm(al)) + assert.Equal(t, placementOptions.ValidZone(), customImpl.opts.ValidZone()) + assert.Equal(t, al, customImpl.algo) +} + func newMockStorage() placement.Storage { return storage.NewPlacementStorage(mem.NewStore(), "", nil) } diff --git a/src/cluster/services/services.go b/src/cluster/services/services.go index 2bf03263cb..7ef98aec7c 100644 --- a/src/cluster/services/services.go +++ b/src/cluster/services/services.go @@ -154,7 +154,7 @@ func (c *client) PlacementService(sid ServiceID, opts placement.Options) (placem return ps.NewPlacementService( storage.NewPlacementStorage(store, c.placementKeyFn(sid), opts), - opts, + ps.WithPlacementOptions(opts), ), nil } diff --git a/src/cmd/services/m3coordinator/downsample/options.go b/src/cmd/services/m3coordinator/downsample/options.go index 2089552c01..b1b61df731 100644 --- a/src/cmd/services/m3coordinator/downsample/options.go +++ b/src/cmd/services/m3coordinator/downsample/options.go @@ -1052,7 +1052,7 @@ func (o DownsamplerOptions) newAggregatorPlacementManager( placementSvc := placementservice.NewPlacementService( placementstorage.NewPlacementStorage(localKVStore, placementKVKey, placementOpts), - placementOpts) + placementservice.WithPlacementOptions(placementOpts)) _, err := placementSvc.BuildInitialPlacement([]placement.Instance{instance}, numShards, replicationFactor) diff --git a/src/msg/integration/setup.go b/src/msg/integration/setup.go index d797077021..0c377a2f46 100644 --- a/src/msg/integration/setup.go +++ b/src/msg/integration/setup.go @@ -536,7 +536,10 @@ func (c *testConsumer) consumeAndAck(totalConsumed *atomic.Int64) { func testPlacementService(store kv.Store, sid services.ServiceID, isSharded bool) placement.Service { opts := placement.NewOptions().SetShardStateMode(placement.StableShardStateOnly).SetIsSharded(isSharded) - return service.NewPlacementService(storage.NewPlacementStorage(store, sid.String(), opts), opts) + + return service.NewPlacementService( + storage.NewPlacementStorage(store, sid.String(), opts), + service.WithPlacementOptions(opts)) } func testProducer( diff --git a/src/msg/producer/writer/consumer_service_writer_test.go b/src/msg/producer/writer/consumer_service_writer_test.go index f9d66e02a1..f88804e3e7 100644 --- a/src/msg/producer/writer/consumer_service_writer_test.go +++ b/src/msg/producer/writer/consumer_service_writer_test.go @@ -608,7 +608,8 @@ func TestConsumerServiceWriterUpdateNonShardedPlacementWithReplicatedConsumption cs := topic.NewConsumerService().SetServiceID(sid).SetConsumptionType(topic.Replicated) sd := services.NewMockServices(ctrl) pOpts := placement.NewOptions().SetIsSharded(false) - ps := service.NewPlacementService(storage.NewPlacementStorage(mem.NewStore(), sid.String(), pOpts), pOpts) + ps := service.NewPlacementService(storage.NewPlacementStorage(mem.NewStore(), sid.String(), pOpts), + service.WithPlacementOptions(pOpts)) sd.EXPECT().PlacementService(sid, gomock.Any()).Return(ps, nil) _, err := ps.BuildInitialPlacement([]placement.Instance{ placement.NewInstance().SetID("i1").SetEndpoint("i1").SetWeight(1), @@ -668,5 +669,7 @@ func TestConsumerServiceCloseShardWritersConcurrently(t *testing.T) { } func testPlacementService(store kv.Store, sid services.ServiceID) placement.Service { - return service.NewPlacementService(storage.NewPlacementStorage(store, sid.String(), placement.NewOptions()), placement.NewOptions()) + return service.NewPlacementService( + storage.NewPlacementStorage(store, sid.String(), placement.NewOptions()), + ) } diff --git a/src/query/api/v1/handler/placement/get_test.go b/src/query/api/v1/handler/placement/get_test.go index 40edc98375..fdae937481 100644 --- a/src/query/api/v1/handler/placement/get_test.go +++ b/src/query/api/v1/handler/placement/get_test.go @@ -70,7 +70,9 @@ func setupPlacementTest(t *testing.T, ctrl *gomock.Controller, initPlacement pla mockClient.EXPECT().Services(gomock.Any()).Return(mockServices, nil).AnyTimes() mockServices.EXPECT().PlacementService(gomock.Any(), gomock.Any()).DoAndReturn( func(_ interface{}, opts placement.Options) (placement.Service, error) { - ps := service.NewPlacementService(storage.NewPlacementStorage(mem.NewStore(), "", opts), opts) + ps := service.NewPlacementService( + storage.NewPlacementStorage(mem.NewStore(), "", opts), + service.WithPlacementOptions(opts)) if initPlacement != nil { _, err := ps.Set(initPlacement) require.NoError(t, err) From aa361f41ecfb4e6240c61b9ce95e3e4e9cd1c9af Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 9 Nov 2020 14:01:27 -0500 Subject: [PATCH 41/47] [lint] Disable nlreturn linter (#2865) Keeping the new line required before a return linter would require a large fraction of the code base to need updating. The perceived benefit of always an empty line does not seem worth the changes required to each file. Also it is purely stylistic and doesn't seem to avoid types of errors. --- .golangci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 5cb97c4887..8ecd89e592 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -212,6 +212,9 @@ linters: # We allow cuddling assignment following conditions because there are valid # logical groupings for this use-case (e.g. when evaluating config values). - wsl + # New line required before return would require a large fraction of the + # code base to need updating, it's not worth the perceived benefit. + - nlreturn disable-all: false presets: # bodyclose, errcheck, gosec, govet, scopelint, staticcheck, typecheck From 5a40a30ec4dde2ed18e4aa63eb2144034b897faa Mon Sep 17 00:00:00 2001 From: Wesley Kim Date: Mon, 9 Nov 2020 14:54:57 -0500 Subject: [PATCH 42/47] Refactor x/lockfile into dbnode/server (#2862) * Refactor x/lockfile into dbnode/server * Remove scripts/lockfile in favor of test case * Convert lockfile APIs / struct to private * Linting fixes + feedback * Add Lockfile suffix to lockfile APIs * Revert go mod changes --- scripts/lockfile/lockfile.go | 65 --------------- src/{x/lockfile => dbnode/server}/lockfile.go | 24 +++--- .../server}/lockfile_test.go | 83 +++++++++++++++---- src/dbnode/server/server.go | 9 +- 4 files changed, 83 insertions(+), 98 deletions(-) delete mode 100644 scripts/lockfile/lockfile.go rename src/{x/lockfile => dbnode/server}/lockfile.go (78%) rename src/{x/lockfile => dbnode/server}/lockfile_test.go (54%) diff --git a/scripts/lockfile/lockfile.go b/scripts/lockfile/lockfile.go deleted file mode 100644 index 4ef44cb5c9..0000000000 --- a/scripts/lockfile/lockfile.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -// This .go file is used to test the lockfile package (m3/src/x/lockfile) - -import ( - "fmt" - "os" - "path" - "strconv" - "time" - - "github.com/m3db/m3/src/x/lockfile" -) - -func exitWithUsage() { - fmt.Printf( - "Usage: %[1]s \nExample: %[1]s /var/run/lockfile 1 1\n", - path.Base(os.Args[0])) - os.Exit(1) -} - -func main() { - if len(os.Args) != 4 { - exitWithUsage() - } - - path, sleepStr, rmLock := os.Args[1], os.Args[2], os.Args[3] - sleep, err := strconv.Atoi(sleepStr) - if err != nil { - exitWithUsage() - } - - lock, err := lockfile.Acquire(path) - if err != nil { - os.Exit(1) - } - - if sleep > 0 { - time.Sleep(time.Duration(sleep) * time.Second) - } - - if rmLock != "0" { - lock.Release() - } -} diff --git a/src/x/lockfile/lockfile.go b/src/dbnode/server/lockfile.go similarity index 78% rename from src/x/lockfile/lockfile.go rename to src/dbnode/server/lockfile.go index bba9d374c5..d678e59d32 100644 --- a/src/x/lockfile/lockfile.go +++ b/src/dbnode/server/lockfile.go @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package lockfile +package server import ( "os" @@ -28,15 +28,15 @@ import ( "golang.org/x/sys/unix" ) -// Lockfile represents an acquired lockfile. -type Lockfile struct { +// lockfile represents an acquired lockfile. +type lockfile struct { file os.File } -// Acquire creates the given file path if it doesn't exist and +// acquireLockfile creates the given file path if it doesn't exist and // obtains an exclusive lock on it. An error is returned if the lock // has been obtained by another process. -func Acquire(path string) (*Lockfile, error) { +func acquireLockfile(path string) (*lockfile, error) { file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666) if err != nil { return nil, errors.Wrap(err, "failed opening lock path") @@ -51,23 +51,23 @@ func Acquire(path string) (*Lockfile, error) { return nil, errors.Wrap(err, "failed obtaining lock") } - lf := Lockfile{*file} + lf := lockfile{*file} return &lf, nil } -// CreateAndAcquire creates any non-existing directories needed to -// create the lock file, then acquires a lock on it -func CreateAndAcquire(path string, newDirMode os.FileMode) (*Lockfile, error) { +// createAndAcquireLockfile creates any non-existing directories needed to +// create the lock file, then acquires a lock on it. +func createAndAcquireLockfile(path string, newDirMode os.FileMode) (*lockfile, error) { if err := os.MkdirAll(paths.Dir(path), newDirMode); err != nil { return nil, err } - return Acquire(path) + return acquireLockfile(path) } -// Release releases the lock on the file and removes the file. -func (lf Lockfile) Release() error { +// releaseLockfile releases the lock on the file and removes the file. +func (lf lockfile) releaseLockfile() error { ft := &unix.Flock_t{ Pid: int32(os.Getpid()), Type: unix.F_UNLCK, diff --git a/src/x/lockfile/lockfile_test.go b/src/dbnode/server/lockfile_test.go similarity index 54% rename from src/x/lockfile/lockfile_test.go rename to src/dbnode/server/lockfile_test.go index 843f5e7135..a0c663b14c 100644 --- a/src/x/lockfile/lockfile_test.go +++ b/src/dbnode/server/lockfile_test.go @@ -18,9 +18,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package lockfile +package server import ( + "fmt" "io/ioutil" "math/rand" "os" @@ -29,6 +30,7 @@ import ( "path/filepath" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -36,32 +38,30 @@ import ( func TestAcquire(t *testing.T) { t.Run("process B can obtain the lock after A exits", func(t *testing.T) { path := tempPath() - assert.NoError(t, newLockfileCommand(path, 0, true).Run()) + assert.NoError(t, newLockfileCommand(path, "", true).Run()) _, err := os.Stat(path) assert.True(t, os.IsNotExist(err)) // check temp file was removed - assert.NoError(t, newLockfileCommand(path, 0, true).Run()) + assert.NoError(t, newLockfileCommand(path, "", true).Run()) }) t.Run("process B can obtain the lock after A exits, even if A didn't remove the lock file", func(t *testing.T) { path := tempPath() - assert.NoError(t, newLockfileCommand(path, 0, false).Run()) + assert.NoError(t, newLockfileCommand(path, "", false).Run()) _, err := os.Stat(path) assert.False(t, os.IsNotExist(err)) // check temp file was *not* removed - assert.NoError(t, newLockfileCommand(path, 0, true).Run()) + assert.NoError(t, newLockfileCommand(path, "", true).Run()) }) t.Run("if process A holds the lock, B must not be able to obtain it", func(t *testing.T) { path := tempPath() - procA := newLockfileCommand(path, 1, true) - procB := newLockfileCommand(path, 1, true) + procA := newLockfileCommand(path, "1s", false) + procB := newLockfileCommand(path, "1s", false) - // to avoid sleeping until A obtains the lock (it takes some - // time for the process to boot and obtain the lock), we start - // both processes, then check exactly one of them failed assert.NoError(t, procA.Start()) assert.NoError(t, procB.Start()) + // one process will acquireLockfile and hold the lock, and the other will fail to acquireLockfile. errA, errB := procA.Wait(), procB.Wait() if errA != nil { @@ -79,25 +79,74 @@ func TestCreateAndAcquire(t *testing.T) { tempSubDir := path.Join(tempDir, "testDir") - lock, err := CreateAndAcquire(path.Join(tempSubDir, "testLockfile"), os.ModePerm) + lock, err := createAndAcquireLockfile(path.Join(tempSubDir, "testLockfile"), os.ModePerm) assert.NoError(t, err) - err = lock.Release() + err = lock.releaseLockfile() assert.NoError(t, err) - // check CreateAndAcquire() created the missing directory + // check createAndAcquireLockfile() created the missing directory _, err = os.Stat(tempSubDir) assert.False(t, os.IsNotExist(err)) } +// TestAcquireAndReleaseFile is invoked as a separate process by other tests in lockfile_test.go +// to exercise the file locking capabilities. The test is a no-op if run as part +// of the broader test suite. Given it's run as a separate process, we explicitly use error +// exit codes as opposed to failing assertions on errors +func TestAcquireAndReleaseFile(t *testing.T) { + // immediately return if this test wasn't invoked by another test in the + // nolint: goconst + if os.Getenv("LOCKFILE_SUPERVISED_PROCESS") != "true" { + t.Skip() + } + + var ( + lockPath = os.Getenv("WITH_LOCK_PATH") + removeLock = os.Getenv("WITH_REMOVE_LOCK") + sleepDuration = os.Getenv("WITH_SLEEP_DURATION") + ) + + lock, err := acquireLockfile(lockPath) + if err != nil { + os.Exit(1) + } + + if sleepDuration != "" { + duration, err := time.ParseDuration(sleepDuration) + if err != nil { + os.Exit(1) + } + + time.Sleep(duration) + } + + if removeLock == "true" { + err := lock.releaseLockfile() + if err != nil { + os.Exit(1) + } + } +} + func tempPath() string { return filepath.Join(os.TempDir(), "lockfile_test_"+strconv.Itoa(os.Getpid())+"_"+strconv.Itoa(rand.Intn(100000))) } -func newLockfileCommand(lockPath string, sleep int, removeLock bool) *exec.Cmd { - removeLockStr := "0" +func newLockfileCommand(lockPath string, sleepDuration string, removeLock bool) *exec.Cmd { + removeLockStr := "false" if removeLock { - removeLockStr = "1" + removeLockStr = "true" } - return exec.Command("go", "run", "../../../scripts/lockfile/lockfile.go", lockPath, strconv.Itoa(sleep), removeLockStr) + cmd := exec.Command("go", "test", "-run", "TestAcquireAndReleaseFile") + cmd.Env = os.Environ() + cmd.Env = append( + cmd.Env, + "LOCKFILE_SUPERVISED_PROCESS=true", + fmt.Sprintf("WITH_LOCK_PATH=%s", lockPath), + fmt.Sprintf("WITH_SLEEP_DURATION=%s", sleepDuration), + fmt.Sprintf("WITH_REMOVE_LOCK=%s", removeLockStr), + ) + + return cmd } diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index dc19aebd6f..114e836bb1 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +// Package server contains the code to run the dbnode server. package server import ( @@ -83,7 +84,6 @@ import ( xdocs "github.com/m3db/m3/src/x/docs" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" - "github.com/m3db/m3/src/x/lockfile" "github.com/m3db/m3/src/x/mmap" xos "github.com/m3db/m3/src/x/os" "github.com/m3db/m3/src/x/pool" @@ -222,11 +222,12 @@ func Run(runOpts RunOptions) { // file will remain on the file system. When a dbnode starts after an ungracefully stop, // it will be able to acquire the lock despite the fact the the lock file exists. lockPath := path.Join(cfg.Filesystem.FilePathPrefixOrDefault(), filePathPrefixLockFile) - fslock, err := lockfile.CreateAndAcquire(lockPath, newDirectoryMode) + fslock, err := createAndAcquireLockfile(lockPath, newDirectoryMode) if err != nil { - logger.Fatal("could not acquire lock", zap.String("path", lockPath), zap.Error(err)) + logger.Fatal("could not acqurie lock", zap.String("path", lockPath), zap.Error(err)) } - defer fslock.Release() + // nolint: errcheck + defer fslock.releaseLockfile() go bgValidateProcessLimits(logger) debug.SetGCPercent(cfg.GCPercentageOrDefault()) From ec68e83b38a7834083a839558fbfaae8c7d61a38 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 9 Nov 2020 16:07:56 -0500 Subject: [PATCH 43/47] Refactor etcd config as discovery section with convenience types (#2843) --- kube/bundle.yaml | 25 +- kube/m3dbnode-configmap.yaml | 25 +- scripts/development/m3_stack/m3dbnode.yml | 29 ++- .../aggregator/m3coordinator.yml | 18 -- .../aggregator_legacy/m3coordinator.yml | 18 -- .../carbon/m3coordinator.yml | 20 -- .../cold_writes_simple/m3coordinator.yml | 20 -- .../m3coordinator.yml | 20 -- .../m3dbnode.yml | 52 +--- .../m3coordinator-cluster-a.yml | 20 -- .../m3coordinator-cluster-b.yml | 20 -- .../m3dbnode-cluster-a.yml | 77 ++---- .../m3dbnode-cluster-b.yml | 79 ++---- .../prometheus/m3coordinator.yml | 20 -- .../m3coordinator01.yml | 20 -- .../m3coordinator02.yml | 20 -- .../query_fanout/m3coordinator-cluster-a.yml | 20 -- .../query_fanout/m3coordinator-cluster-b.yml | 21 -- .../query_fanout/m3coordinator-cluster-c.yml | 20 -- .../repair/m3coordinator.yml | 20 -- .../repair/m3dbnode.yml | 76 +----- .../m3coordinator-cluster-a.yml | 5 - .../m3coordinator-cluster-b.yml | 20 -- .../m3dbnode-cluster-a.yml | 79 ++---- .../m3dbnode-cluster-b.yml | 75 +----- .../replication/m3coordinator-cluster-a.yml | 20 -- .../replication/m3coordinator-cluster-b.yml | 20 -- .../replication/m3dbnode-cluster-a.yml | 76 +----- .../replication/m3dbnode-cluster-b.yml | 76 +----- .../simple_v2_batch_apis/m3coordinator.yml | 20 -- src/cmd/services/m3dbnode/config/config.go | 15 +- .../services/m3dbnode/config/config_test.go | 185 +++++++------- .../services/m3dbnode/main/main_index_test.go | 26 +- src/cmd/services/m3dbnode/main/main_test.go | 80 +++--- .../m3query/config/testdata/config_test.yml | 32 --- .../resources/config/m3coordinator.yml | 20 -- .../harness/resources/config/m3dbnode.yml | 31 +-- src/dbnode/config/m3dbnode-all-config.yml | 41 +-- .../config/m3dbnode-cluster-template.yml | 60 ----- .../config/m3dbnode-local-etcd-proto.yml | 94 +------ src/dbnode/config/m3dbnode-local-etcd.yml | 16 +- src/dbnode/discovery/config.go | 235 ++++++++++++++++++ src/dbnode/discovery/config_test.go | 166 +++++++++++++ src/dbnode/server/server.go | 31 ++- .../config/m3coordinator-cluster-template.yml | 18 -- src/query/config/m3coordinator-local-etcd.yml | 20 -- src/query/config/m3query-dev-etcd.yml | 42 ---- src/query/config/m3query-local-etcd.yml | 41 --- src/query/server/query.go | 29 ++- 49 files changed, 803 insertions(+), 1410 deletions(-) create mode 100644 src/dbnode/discovery/config.go create mode 100644 src/dbnode/discovery/config_test.go diff --git a/kube/bundle.yaml b/kube/bundle.yaml index 8449fcf925..bc2bd61b36 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -175,18 +175,19 @@ data: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - http://etcd-0.etcd:2379 - - http://etcd-1.etcd:2379 - - http://etcd-2.etcd:2379 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd-0.etcd:2379 + - http://etcd-1.etcd:2379 + - http://etcd-2.etcd:2379 --- # Headless service for the statefulset apiVersion: v1 diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 6ccd3084e3..0536215c4a 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -65,15 +65,16 @@ data: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - http://etcd-0.etcd:2379 - - http://etcd-1.etcd:2379 - - http://etcd-2.etcd:2379 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd-0.etcd:2379 + - http://etcd-1.etcd:2379 + - http://etcd-2.etcd:2379 diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 35a384a193..fb3db8d151 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -12,20 +12,25 @@ db: resolver: environment envVarName: M3DB_HOST_ID - config: + # Note: cannot use type: "m3db_single_node" since sometimes + # multiple DB nodes spawned using the m3_stack start script + # and as such the non-seed nodes need to point etcd not to + # localhost but to m3db_seed:2379 specifically. + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - m3db_seed:2379 - seedNodes: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - m3db_seed:2379 + seedNodes: initialCluster: - - hostID: m3db_seed - endpoint: http://m3db_seed:2380 + - hostID: m3db_seed + endpoint: http://m3db_seed:2380 # proto: # schemaFilePath: /etc/m3dbnode/schema.proto diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index 35a42c248f..8d3afe98b6 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -1,21 +1,5 @@ listenAddress: 0.0.0.0:7202 -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - carbon: ingester: listenAddress: "0.0.0.0:7204" @@ -47,8 +31,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index db8feefcd7..3649b73580 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -1,21 +1,5 @@ listenAddress: 0.0.0.0:7202 -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - carbon: ingester: listenAddress: "0.0.0.0:7204" @@ -47,8 +31,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index ba69263bc6..a883cdbc0c 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -33,8 +18,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -56,6 +39,3 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index 64ae45cdca..37ff9ff20f 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 1ad54297c1..46a9f4cdc7 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -41,8 +26,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: rules: @@ -77,6 +60,3 @@ downsample: bufferPastLimits: - resolution: 0s bufferPast: 90s - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index 23e7bdf9eb..11795536c2 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -18,56 +18,14 @@ coordinator: idScheme: quoted db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: + discovery: + type: m3db_cluster + m3dbCluster: env: foo-namespace/foo-cluster zone: bar-zone - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: bar-zone - endpoints: - - etcd01:2379 + endpoints: + - etcd01:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index 1a165ab6b9..9435c4aaf6 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 124973365a..13aac06336 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -1,36 +1,9 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority config: services: - service: @@ -53,38 +26,18 @@ db: - cluster_b_dbnode01:2379 async: true - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index e1c13baba6..9e59d22898 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -1,69 +1,20 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index f9d46f31b3..92edee33fc 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,11 +22,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted query: restrictTags: diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 83d61c1b74..690e81eac6 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - writeForwarding: promRemoteWrite: targets: @@ -38,8 +23,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index 8d15b050d8..b593362683 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -33,8 +18,3 @@ clusters: - zone: embedded endpoints: - dbnode02:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml index a3ef780416..85bf17a51d 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-a:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -55,9 +38,6 @@ carbon: - resolution: 5s retention: 10h -tagOptions: - idScheme: quoted - # Use tag consolidation here; other integration tests handle id consolidations. query: consolidation: diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml index 2464d98685..07be9bbc8e 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-b:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -54,10 +37,6 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted - query: consolidation: matchType: tags \ No newline at end of file diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml index 034036b6ef..21e70608e7 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-c:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -55,9 +38,6 @@ carbon: - resolution: 5s retention: 10h -tagOptions: - idScheme: quoted - query: consolidation: matchType: tags \ No newline at end of file diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index 64ae45cdca..37ff9ff20f 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index eb307f1e81..29dcc22d4b 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -1,76 +1,26 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - dbnode01:2379 seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 + initialCluster: + - hostID: m3db_local_1 + endpoint: http://dbnode01:2380 # Enable repairs. repair: enabled: true throttle: 1ms checkInterval: 1ms - diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml index 1a165ab6b9..fdfefaeb3a 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml @@ -37,8 +37,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index f2403c8890..3c3b91f4bf 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 # Enable repairs (within cluster a). repair: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index bcdd21cb21..920ab4db81 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 # Enable repairs (within cluster b). repair: diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index 1a165ab6b9..9435c4aaf6 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index e89976d488..52e75125c5 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 # Disable repairs (within cluster a). repair: @@ -90,4 +41,3 @@ db: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 195497e169..2d04a09719 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 # Disable repairs (within cluster b). repair: @@ -92,4 +43,3 @@ db: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 68f407d984..83e9cf4df4 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,9 +22,4 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority useV2BatchAPIs: true - -tagOptions: - idScheme: quoted diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index 9f438cec6b..1ec7e0a165 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -31,6 +31,7 @@ import ( coordinatorcfg "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/dbnode/client" + "github.com/m3db/m3/src/dbnode/discovery" "github.com/m3db/m3/src/dbnode/environment" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/x/config/hostid" @@ -164,8 +165,8 @@ type DBConfiguration struct { // The pooling policy. PoolingPolicy *PoolingPolicy `yaml:"pooling"` - // The environment (static or dynamic) configuration. - EnvironmentConfig environment.Configuration `yaml:"config"` + // The discovery configuration. + DiscoveryConfig discovery.Configuration `yaml:"discovery"` // The configuration for hashing Hashing HashingConfiguration `yaml:"hashing"` @@ -587,12 +588,18 @@ func (c *ProtoConfiguration) Validate() error { // NewEtcdEmbedConfig creates a new embedded etcd config from kv config. func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error) { newKVCfg := embed.NewConfig() - kvCfg := cfg.EnvironmentConfig.SeedNodes hostID, err := cfg.HostID.Resolve() if err != nil { - return nil, err + return nil, fmt.Errorf("failed resolving hostID %w", err) } + + envCfg, err := cfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + return nil, fmt.Errorf("failed getting env config from discovery config %w", err) + } + + kvCfg := envCfg.SeedNodes newKVCfg.Name = hostID dir := kvCfg.RootDir diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 2db737a5df..5c98086e00 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -274,37 +274,38 @@ db: lowWatermark: 0.01 highWatermark: 0.02 - config: - service: - env: production - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 1.1.1.1:2379 - - 1.1.1.2:2379 - - 1.1.1.3:2379 - - seedNodes: - listenPeerUrls: - - http://0.0.0.0:2380 - listenClientUrls: - - http://0.0.0.0:2379 - rootDir: /var/lib/etcd - initialAdvertisePeerUrls: - - http://1.1.1.1:2380 - advertiseClientUrls: - - http://1.1.1.1:2379 - initialCluster: - - hostID: host1 - endpoint: http://1.1.1.1:2380 - clusterState: existing - - hostID: host2 - endpoint: http://1.1.1.2:2380 - - hostID: host3 - endpoint: http://1.1.1.3:2380 + discovery: + config: + service: + env: production + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 1.1.1.1:2379 + - 1.1.1.2:2379 + - 1.1.1.3:2379 + + seedNodes: + listenPeerUrls: + - http://0.0.0.0:2380 + listenClientUrls: + - http://0.0.0.0:2379 + rootDir: /var/lib/etcd + initialAdvertisePeerUrls: + - http://1.1.1.1:2380 + advertiseClientUrls: + - http://1.1.1.1:2379 + initialCluster: + - hostID: host1 + endpoint: http://1.1.1.1:2380 + clusterState: existing + - hostID: host2 + endpoint: http://1.1.1.2:2380 + - hostID: host3 + endpoint: http://1.1.1.3:2380 hashing: seed: 42 writeNewSeriesAsync: true @@ -602,65 +603,69 @@ func TestConfiguration(t *testing.T) { size: 8 lowWatermark: 0 highWatermark: 0 - config: - services: - - async: false - clientOverrides: - hostQueueFlushInterval: null - targetHostQueueFlushSize: null - service: - zone: embedded - env: production - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 1.1.1.1:2379 - - 1.1.1.2:2379 - - 1.1.1.3:2379 - keepAlive: null - tls: null - autoSyncInterval: 0s - m3sd: - initTimeout: null - watchWithRevision: 0 - newDirectoryMode: null - statics: [] - seedNodes: - rootDir: /var/lib/etcd - initialAdvertisePeerUrls: - - http://1.1.1.1:2380 - advertiseClientUrls: - - http://1.1.1.1:2379 - listenPeerUrls: - - http://0.0.0.0:2380 - listenClientUrls: - - http://0.0.0.0:2379 - initialCluster: - - hostID: host1 - endpoint: http://1.1.1.1:2380 - clusterState: existing - - hostID: host2 - endpoint: http://1.1.1.2:2380 - clusterState: "" - - hostID: host3 - endpoint: http://1.1.1.3:2380 - clusterState: "" - clientTransportSecurity: - caFile: "" - certFile: "" - keyFile: "" - trustedCaFile: "" - clientCertAuth: false - autoTls: false - peerTransportSecurity: - caFile: "" - certFile: "" - keyFile: "" - trustedCaFile: "" - clientCertAuth: false - autoTls: false + discovery: + type: null + m3dbCluster: null + m3AggregatorCluster: null + config: + services: + - async: false + clientOverrides: + hostQueueFlushInterval: null + targetHostQueueFlushSize: null + service: + zone: embedded + env: production + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 1.1.1.1:2379 + - 1.1.1.2:2379 + - 1.1.1.3:2379 + keepAlive: null + tls: null + autoSyncInterval: 0s + m3sd: + initTimeout: null + watchWithRevision: 0 + newDirectoryMode: null + statics: [] + seedNodes: + rootDir: /var/lib/etcd + initialAdvertisePeerUrls: + - http://1.1.1.1:2380 + advertiseClientUrls: + - http://1.1.1.1:2379 + listenPeerUrls: + - http://0.0.0.0:2380 + listenClientUrls: + - http://0.0.0.0:2379 + initialCluster: + - hostID: host1 + endpoint: http://1.1.1.1:2380 + clusterState: existing + - hostID: host2 + endpoint: http://1.1.1.2:2380 + clusterState: "" + - hostID: host3 + endpoint: http://1.1.1.3:2380 + clusterState: "" + clientTransportSecurity: + caFile: "" + certFile: "" + keyFile: "" + trustedCaFile: "" + clientCertAuth: false + autoTls: false + peerTransportSecurity: + caFile: "" + certFile: "" + keyFile: "" + trustedCaFile: "" + clientCertAuth: false + autoTls: false hashing: seed: 42 writeNewSeriesAsync: true diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index 276c01bfde..bd302eba66 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -111,7 +111,10 @@ func TestIndexEnabledServer(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -193,7 +196,7 @@ func TestIndexEnabledServer(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -447,13 +450,14 @@ db: - capacity: 4096 size: 128 - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: {{.EtcdEndpoints}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: {{.EtcdEndpoints}} ` diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index 0bb417b2d7..ccfafbf771 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -103,7 +103,10 @@ func TestConfig(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -185,7 +188,7 @@ func TestConfig(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -334,7 +337,10 @@ func TestEmbeddedConfig(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -395,7 +401,7 @@ func TestEmbeddedConfig(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -613,40 +619,42 @@ db: ` kvConfigPortion = ` - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: {{.EtcdEndpoints}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: {{.EtcdEndpoints}} ` embeddedKVConfigPortion = ` - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: - - {{.EtcdEndpoint}} - seedNodes: - rootDir: {{.EmbeddedKVDir}} - listenPeerUrls: - - {{.LPURL}} - listenClientUrls: - - {{.LCURL}} - initialAdvertisePeerUrls: - - {{.APURL}} - advertiseClientUrls: - - {{.ACURL}} - initialCluster: - - hostID: {{.InitialClusterHostID}} - endpoint: {{.InitialClusterEndpoint}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: + - {{.EtcdEndpoint}} + seedNodes: + rootDir: {{.EmbeddedKVDir}} + listenPeerUrls: + - {{.LPURL}} + listenClientUrls: + - {{.LCURL}} + initialAdvertisePeerUrls: + - {{.APURL}} + advertiseClientUrls: + - {{.ACURL}} + initialCluster: + - hostID: {{.InitialClusterHostID}} + endpoint: {{.InitialClusterEndpoint}} ` ) diff --git a/src/cmd/services/m3query/config/testdata/config_test.yml b/src/cmd/services/m3query/config/testdata/config_test.yml index 68250c3f0d..dbe0cbb233 100644 --- a/src/cmd/services/m3query/config/testdata/config_test.yml +++ b/src/cmd/services/m3query/config/testdata/config_test.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -33,23 +18,6 @@ clusters: initialCluster: - hostID: m3db_local endpoint: http://127.0.0.1:2380 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 limits: perQuery: diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml index 53764669f0..536499f031 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: aggregated @@ -33,8 +18,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -56,6 +39,3 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 193ec26aa4..28627c78f7 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -70,21 +70,22 @@ db: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 - + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 127.0.0.1:2379 + seedNodes: + initialCluster: + - hostID: m3db_local + endpoint: http://127.0.0.1:2380 + # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ # for quick local setup (which this config will send data to). diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 51d08e4812..da6caf8692 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -144,23 +144,24 @@ db: checkInterval: 1m # etcd configuration. - config: - service: - # KV environment, zone, and service from which to write/read KV data (placement - # and configuration). Leave these as the default values unless you know what - # you're doing. - env: default_env - zone: embedded - service: m3db - # Directory to store cached etcd data in. - cacheDir: /var/lib/m3kv - # Configuration to identify the etcd hosts this node should connect to. - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - # Should only be present if running an M3DB cluster with embedded etcd. - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + config: + service: + # KV environment, zone, and service from which to write/read KV data (placement + # and configuration). Leave these as the default values unless you know what + # you're doing. + env: default_env + zone: embedded + service: m3db + # Directory to store cached etcd data in. + cacheDir: /var/lib/m3kv + # Configuration to identify the etcd hosts this node should connect to. + etcdClusters: + - zone: embedded + endpoints: + - 127.0.0.1:2379 + # Should only be present if running an M3DB cluster with embedded etcd. + seedNodes: + initialCluster: + - hostID: m3db_local + endpoint: http://127.0.0.1:2380 diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index fe4de58051..4d6689a84c 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -1,40 +1,11 @@ coordinator: - listenAddress: 0.0.0.0:7201 - local: namespaces: - namespace: default type: unaggregated retention: 48h - logging: - level: info - - metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - - tagOptions: - # Configuration setting for generating metric IDs from tags. - idScheme: quoted - db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - hostID: resolver: hostname @@ -59,34 +30,3 @@ db: # endpoint: http://HOST2_STATIC_IP_ADDRESS:2380 # - hostID: host3 # endpoint: http://HOST3_STATIC_IP_ADDRESS:2380 - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index e76bb65f0e..3b49b3574f 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -1,100 +1,22 @@ coordinator: - listenAddress: 0.0.0.0:7201 - local: namespaces: - namespace: default type: unaggregated retention: 48h - logging: - level: info - - metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - - tagOptions: - # Configuration setting for generating metric IDs from tags. - idScheme: quoted - db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: config value: m3db_local - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + type: m3db_single_node proto: - enabled: true - schema_registry: - # Need an entry for each configured namespace. - "default": - schemaFilePath: "/etc/m3dbnode/default_schema.proto" - messageName: "VehicleLocation" - - # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ - # for quick local setup (which this config will send data to). - - # tracing: - # backend: jaeger + enabled: true + schema_registry: + # Need an entry for each configured namespace. + "default": + schemaFilePath: "/etc/m3dbnode/default_schema.proto" + messageName: "VehicleLocation" diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index b40dbaae7d..51019f02d4 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -10,20 +10,8 @@ db: resolver: config value: m3db_local - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + type: m3db_single_node # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ # for quick local setup (which this config will send data to). diff --git a/src/dbnode/discovery/config.go b/src/dbnode/discovery/config.go new file mode 100644 index 0000000000..8cb86318a1 --- /dev/null +++ b/src/dbnode/discovery/config.go @@ -0,0 +1,235 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package discovery provides discovery configuration. +package discovery + +import ( + "fmt" + + etcdclient "github.com/m3db/m3/src/cluster/client/etcd" + "github.com/m3db/m3/src/dbnode/environment" +) + +const ( + defaultEnvironment = "default_env" + defaultZone = "embedded" + defaultM3DBService = "m3db" + defaultM3AggregatorService = "m3aggregator" + defaultCacheDirectory = "/var/lib/m3kv" + defaultSingleNodeClusterEndpoint = "127.0.0.1:2379" + defaultSingleNodeClusterSeedEndpoint = "http://127.0.0.1:2380" +) + +var validDiscoveryConfigTypes = []ConfigurationType{ + ConfigType, + M3DBSingleNodeType, + M3DBClusterType, + M3AggregatorClusterType, +} + +// ConfigurationType defines the type of discovery configuration. +type ConfigurationType uint + +const ( + // ConfigType defines a generic definition for service discovery via etcd. + ConfigType ConfigurationType = iota + // M3DBSingleNodeType defines configuration for a single M3DB node via etcd. + M3DBSingleNodeType + // M3DBClusterType defines M3DB discovery via etcd. + M3DBClusterType + // M3AggregatorClusterType defines M3DB discovery via etcd. + M3AggregatorClusterType +) + +// UnmarshalYAML unmarshals an ConfigurationType into a valid type from string. +func (t *ConfigurationType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + // If unspecified, use default mode. + if str == "" { + *t = ConfigType + + return nil + } + + for _, valid := range validDiscoveryConfigTypes { + if str == valid.String() { + *t = valid + + return nil + } + } + + return fmt.Errorf("invalid ConfigurationType '%s' valid types are: %s", + str, validDiscoveryConfigTypes) +} + +// String returns the discovery configuration type as a string. +func (t ConfigurationType) String() string { + switch t { + case ConfigType: + return "config" + case M3DBSingleNodeType: + return "m3db_single_node" + case M3DBClusterType: + return "m3db_cluster" + case M3AggregatorClusterType: + return "m3aggregator_cluster" + } + return "unknown" +} + +// Configuration defines how services are to be discovered. +type Configuration struct { + // Type defines the type of discovery configuration being used. + Type *ConfigurationType `yaml:"type"` + + // M3DBCluster defines M3DB discovery via etcd. + M3DBCluster *M3DBClusterDiscoveryConfiguration `yaml:"m3dbCluster"` + + // M3AggregatorCluster defines M3Aggregator discovery via etcd. + M3AggregatorCluster *M3AggregatorClusterDiscoveryConfiguration `yaml:"m3AggregatorCluster"` + + // Config defines a generic definition for service discovery via etcd. + Config *environment.Configuration `yaml:"config"` +} + +// M3DBClusterDiscoveryConfiguration defines discovery configuration for M3DB. +type M3DBClusterDiscoveryConfiguration struct { + Env string `yaml:"env" validate:"nonzero"` + Zone *string `yaml:"zone"` + Endpoints []string `yaml:"endpoints"` +} + +// M3AggregatorClusterDiscoveryConfiguration defines discovery configuration for M3Aggregator. +type M3AggregatorClusterDiscoveryConfiguration struct { + Env string `yaml:"env"` + Zone *string `yaml:"zone"` + Endpoints []string `yaml:"endpoints"` +} + +// EnvironmentConfig provides the environment configuration +// based on the type of discovery configuration set. +func (c *Configuration) EnvironmentConfig( + hostID string, +) (environment.Configuration, error) { + discoveryConfigType := ConfigType + if c.Type != nil { + discoveryConfigType = *c.Type + } + + switch discoveryConfigType { + case ConfigType: + return *c.Config, nil + case M3DBSingleNodeType: + return c.m3dbSingleNodeEnvConfig(hostID), nil + case M3DBClusterType: + return c.envConfig( + discoveryConfigType, + defaultM3DBService, + c.M3DBCluster.Zone, + c.M3DBCluster.Env, + c.M3DBCluster.Endpoints, + ) + case M3AggregatorClusterType: + return c.envConfig( + discoveryConfigType, + defaultM3AggregatorService, + c.M3AggregatorCluster.Zone, + c.M3AggregatorCluster.Env, + c.M3AggregatorCluster.Endpoints, + ) + } + + return environment.Configuration{}, fmt.Errorf("unrecognized discovery type: %d", c.Type) +} + +func (c *Configuration) m3dbSingleNodeEnvConfig( + hostID string, +) environment.Configuration { + return environment.Configuration{ + Services: []*environment.DynamicCluster{ + { + Service: &etcdclient.Configuration{ + Service: defaultM3DBService, + CacheDir: defaultCacheDirectory, + Zone: defaultZone, + Env: defaultEnvironment, + ETCDClusters: []etcdclient.ClusterConfig{ + { + Zone: defaultZone, + Endpoints: []string{defaultSingleNodeClusterEndpoint}, + }, + }, + }, + }, + }, + SeedNodes: &environment.SeedNodesConfig{ + InitialCluster: []environment.SeedNode{ + { + HostID: hostID, + Endpoint: defaultSingleNodeClusterSeedEndpoint, + }, + }, + }, + } +} + +func (c *Configuration) envConfig( + configType ConfigurationType, + service string, + zone *string, + env string, + endpoints []string, +) (environment.Configuration, error) { + if c == nil { + err := fmt.Errorf("discovery configuration required for type: %s", + configType.String()) + return environment.Configuration{}, err + } + + validZone := defaultZone + if zone != nil { + validZone = *zone + } + + return environment.Configuration{ + Services: []*environment.DynamicCluster{ + { + Service: &etcdclient.Configuration{ + Service: service, + CacheDir: defaultCacheDirectory, + Zone: validZone, + Env: env, + ETCDClusters: []etcdclient.ClusterConfig{ + { + Zone: validZone, + Endpoints: endpoints, + }, + }, + }, + }, + }, + }, nil +} diff --git a/src/dbnode/discovery/config_test.go b/src/dbnode/discovery/config_test.go new file mode 100644 index 0000000000..307769688c --- /dev/null +++ b/src/dbnode/discovery/config_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package discovery + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/m3db/m3/src/dbnode/environment" + "github.com/m3db/m3/src/x/config" +) + +func TestM3DBSingleNodeType(t *testing.T) { + in := ` +type: m3db_single_node +` + + hostID := "test_id" + envConfig := getEnvConfig(t, in, hostID) + + assert.Equal(t, 1, len(envConfig.Services)) + assert.Equal(t, 1, len(envConfig.SeedNodes.InitialCluster)) + + s := envConfig.Services[0].Service + assert.Equal(t, defaultM3DBService, s.Service) + assert.Equal(t, defaultEnvironment, s.Env) + assert.Equal(t, defaultZone, s.Zone) + assert.Equal(t, defaultCacheDirectory, s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, defaultZone, s.ETCDClusters[0].Zone) + assert.Equal(t, 1, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, defaultSingleNodeClusterEndpoint, s.ETCDClusters[0].Endpoints[0]) + + c := envConfig.SeedNodes.InitialCluster[0] + assert.Equal(t, defaultSingleNodeClusterSeedEndpoint, c.Endpoint) + assert.Equal(t, hostID, c.HostID) +} + +func TestM3DBClusterType(t *testing.T) { + in := ` +type: m3db_cluster +m3dbCluster: + env: a + zone: b + endpoints: + - end_1 + - end_2 +` + + envConfig := getEnvConfig(t, in, "") + validateClusterConfig(t, envConfig, defaultM3DBService) +} + +func TestM3AggregatorClusterType(t *testing.T) { + in := ` +type: m3aggregator_cluster +m3AggregatorCluster: + env: a + zone: b + endpoints: + - end_1 + - end_2 +` + + envConfig := getEnvConfig(t, in, "") + validateClusterConfig(t, envConfig, defaultM3AggregatorService) +} + +func TestConfigType(t *testing.T) { + in := ` +config: + service: + env: test_env + zone: test_zone + service: test_service + cacheDir: test/cache + etcdClusters: + - zone: test_zone_2 + endpoints: + - 127.0.0.1:2379 + seedNodes: + initialCluster: + - hostID: host_id + endpoint: http://127.0.0.1:2380 +` + + hostID := "test_id" + envConfig := getEnvConfig(t, in, hostID) + + assert.Equal(t, 1, len(envConfig.Services)) + assert.Equal(t, 1, len(envConfig.SeedNodes.InitialCluster)) + + s := envConfig.Services[0].Service + assert.Equal(t, "test_service", s.Service) + assert.Equal(t, "test_env", s.Env) + assert.Equal(t, "test_zone", s.Zone) + assert.Equal(t, "test/cache", s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, "test_zone_2", s.ETCDClusters[0].Zone) + assert.Equal(t, 1, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, "127.0.0.1:2379", s.ETCDClusters[0].Endpoints[0]) + + c := envConfig.SeedNodes.InitialCluster[0] + assert.Equal(t, "http://127.0.0.1:2380", c.Endpoint) + assert.Equal(t, "host_id", c.HostID) +} + +func getEnvConfig(t *testing.T, in string, hostID string) environment.Configuration { + fd, err := ioutil.TempFile("", "config.yaml") + assert.NoError(t, err) + defer func() { + assert.NoError(t, fd.Close()) + assert.NoError(t, os.Remove(fd.Name())) + }() + + _, err = fd.Write([]byte(in)) + assert.NoError(t, err) + + var cfg Configuration + err = config.LoadFile(&cfg, fd.Name(), config.Options{}) + assert.NoError(t, err) + + envConfig, err := cfg.EnvironmentConfig(hostID) + assert.NoError(t, err) + + return envConfig +} + +func validateClusterConfig(t *testing.T, + envConfig environment.Configuration, + expectedService string, +) { + assert.Equal(t, 1, len(envConfig.Services)) + assert.Nil(t, envConfig.SeedNodes) + s := envConfig.Services[0].Service + assert.Equal(t, expectedService, s.Service) + assert.Equal(t, "a", s.Env) + assert.Equal(t, "b", s.Zone) + assert.Equal(t, defaultCacheDirectory, s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, "b", s.ETCDClusters[0].Zone) + assert.Equal(t, 2, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, "end_1", s.ETCDClusters[0].Endpoints[0]) + assert.Equal(t, "end_2", s.ETCDClusters[0].Endpoints[1]) +} diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 114e836bb1..3e619835ca 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -268,17 +268,22 @@ func Run(runOpts RunOptions) { } // Presence of KV server config indicates embedded etcd cluster - if cfg.EnvironmentConfig.SeedNodes == nil { + envConfig, err := cfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", zap.Error(err)) + } + + if envConfig.SeedNodes == nil { logger.Info("no seed nodes set, using dedicated etcd cluster") } else { // Default etcd client clusters if not set already - service, err := cfg.EnvironmentConfig.Services.SyncCluster() + service, err := envConfig.Services.SyncCluster() if err != nil { logger.Fatal("invalid cluster configuration", zap.Error(err)) } clusters := service.Service.ETCDClusters - seedNodes := cfg.EnvironmentConfig.SeedNodes.InitialCluster + seedNodes := envConfig.SeedNodes.InitialCluster if len(clusters) == 0 { endpoints, err := config.InitialClusterEndpoints(seedNodes) if err != nil { @@ -608,12 +613,12 @@ func Run(runOpts RunOptions) { opts = opts.SetPersistManager(pm) var ( - envCfg environment.ConfigureResults + envCfgResults environment.ConfigureResults ) - if len(cfg.EnvironmentConfig.Statics) == 0 { + if len(envConfig.Statics) == 0 { logger.Info("creating dynamic config service client with m3cluster") - envCfg, err = cfg.EnvironmentConfig.Configure(environment.ConfigurationParameters{ + envCfgResults, err = envConfig.Configure(environment.ConfigurationParameters{ InstrumentOpts: iopts, HashingSeed: cfg.Hashing.Seed, NewDirectoryMode: newDirectoryMode, @@ -625,7 +630,7 @@ func Run(runOpts RunOptions) { } else { logger.Info("creating static config service client with m3cluster") - envCfg, err = cfg.EnvironmentConfig.Configure(environment.ConfigurationParameters{ + envCfgResults, err = envConfig.Configure(environment.ConfigurationParameters{ InstrumentOpts: iopts, HostID: hostID, ForceColdWritesEnabled: runOpts.StorageOptions.ForceColdWritesEnabled, @@ -635,7 +640,7 @@ func Run(runOpts RunOptions) { } } - syncCfg, err := envCfg.SyncCluster() + syncCfg, err := envCfgResults.SyncCluster() if err != nil { logger.Fatal("invalid cluster config", zap.Error(err)) } @@ -704,11 +709,11 @@ func Run(runOpts RunOptions) { if err != nil { logger.Warn("could not create handler options for debug writer", zap.Error(err)) } else { - envCfg, err := cfg.EnvironmentConfig.Services.SyncCluster() - if err != nil || envCfg.Service == nil { + envCfgCluster, err := envConfig.Services.SyncCluster() + if err != nil || envCfgCluster.Service == nil { logger.Warn("could not get cluster config for debug writer", zap.Error(err), - zap.Bool("envCfgServiceIsNil", envCfg.Service == nil)) + zap.Bool("envCfgClusterServiceIsNil", envCfgCluster.Service == nil)) } else { debugWriter, err = xdebug.NewPlacementAndNamespaceZipWriterWithDefaultSources( cpuProfileDuration, @@ -718,8 +723,8 @@ func Run(runOpts RunOptions) { { ServiceName: handleroptions.M3DBServiceName, Defaults: []handleroptions.ServiceOptionsDefault{ - handleroptions.WithDefaultServiceEnvironment(envCfg.Service.Env), - handleroptions.WithDefaultServiceZone(envCfg.Service.Zone), + handleroptions.WithDefaultServiceEnvironment(envCfgCluster.Service.Env), + handleroptions.WithDefaultServiceZone(envCfgCluster.Service.Zone), }, }, }, diff --git a/src/query/config/m3coordinator-cluster-template.yml b/src/query/config/m3coordinator-cluster-template.yml index 79887d3388..bed8f8cae3 100644 --- a/src/query/config/m3coordinator-cluster-template.yml +++ b/src/query/config/m3coordinator-cluster-template.yml @@ -1,21 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - clusters: ## Fill-out the following and un-comment before using, and ## make sure indent by two spaces is applied. diff --git a/src/query/config/m3coordinator-local-etcd.yml b/src/query/config/m3coordinator-local-etcd.yml index 0dfefbd45f..1fb1940638 100644 --- a/src/query/config/m3coordinator-local-etcd.yml +++ b/src/query/config/m3coordinator-local-etcd.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -29,8 +14,3 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/src/query/config/m3query-dev-etcd.yml b/src/query/config/m3query-dev-etcd.yml index cdaf363ce9..90a62af0d9 100644 --- a/src/query/config/m3query-dev-etcd.yml +++ b/src/query/config/m3query-dev-etcd.yml @@ -1,21 +1,5 @@ # m3query configuration for local development setup. Mostly the same as m3query-local-etcd.yml, but using fewer # resources (threads primarily). - -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -32,23 +16,6 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 readWorkerPoolPolicy: grow: false @@ -57,12 +24,3 @@ readWorkerPoolPolicy: writeWorkerPoolPolicy: grow: false size: 10 - -tagOptions: - idScheme: quoted - -# Uncomment this to enable local jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ -# for quick local setup (which this config will send data to). - -# tracing: -# backend: jaeger diff --git a/src/query/config/m3query-local-etcd.yml b/src/query/config/m3query-local-etcd.yml index f63b801a41..1fb1940638 100644 --- a/src/query/config/m3query-local-etcd.yml +++ b/src/query/config/m3query-local-etcd.yml @@ -1,21 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - clusters: - namespaces: - namespace: default @@ -32,26 +14,3 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 - -# Uncomment this to enable local jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ -# for quick local setup (which this config will send data to). - -# tracing: -# backend: jaeger diff --git a/src/query/server/query.go b/src/query/server/query.go index c67327c669..0d44c0bbc4 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -436,7 +436,19 @@ func Run(runOpts RunOptions) { var serviceOptionDefaults []handleroptions.ServiceOptionsDefault if dbCfg := runOpts.DBConfig; dbCfg != nil { - cluster, err := dbCfg.EnvironmentConfig.Services.SyncCluster() + hostID, err := dbCfg.HostID.Resolve() + if err != nil { + logger.Fatal("could not resolve hostID", + zap.Error(err)) + } + + envCfg, err := dbCfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", + zap.Error(err)) + } + + cluster, err := envCfg.Services.SyncCluster() if err != nil { logger.Fatal("could not resolve embedded db cluster info", zap.Error(err)) @@ -818,7 +830,20 @@ func initClusters( if dbCfg == nil { return nil, nil, nil, errors.New("environment config required when dynamically fetching namespaces") } - clusterStaticConfig.Client = client.Configuration{EnvironmentConfig: &dbCfg.EnvironmentConfig} + + hostID, err := dbCfg.HostID.Resolve() + if err != nil { + logger.Fatal("could not resolve hostID", + zap.Error(err)) + } + + envCfg, err := dbCfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", + zap.Error(err)) + } + + clusterStaticConfig.Client = client.Configuration{EnvironmentConfig: &envCfg} } clustersCfg := m3.ClustersStaticConfiguration{clusterStaticConfig} From 85b193b24a4e0b61ebe06d14153bd6020ccadc75 Mon Sep 17 00:00:00 2001 From: Matt Way Date: Mon, 9 Nov 2020 16:11:46 -0500 Subject: [PATCH 44/47] Add COMPATIBILITY.md to describe version compatibility (#2829) --- COMPATIBILITY.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 COMPATIBILITY.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 0000000000..8c8ba40971 --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,25 @@ +# Compatibility + +This document describes versioning compatibility guarantees. Any compatibility +outlined here will be maintained for at least the current major version. +Additional guarantees may be added within the same major version, but existing +guarantees may only be modified or removed as part of a major version change. + +If you discover a breaking change, please [open an issue][issue]. + +[issue]: https://github.com/m3db/m3/issues/new + +## v1.0 + +As of version 1.0, m3db/m3 and its components are guaranteed to have **binary**, +**configuration**, **wire**, and **transport API** compatibility: + +| Compatibility | Scope | Guarantee | +| :------------ | :---- | :-------- | +| binary | All released executable components. | All executable components will maintain their current functionality. Components may have *additional* functionality added, but no existing functionality will be changed or removed. | +| configuration | All configuration serialization for binary components. | All configuration will maintain serialization compatibility. New configuration properties may be added, but existing properties will not be renamed, removed, re-typed, or changed semantically or behaviorally. | +| wire | All on-the-wire encoding for data to/from binary components. | All wire encoding (e.g. Protobuf, Thrift) will maintain wire compatibility. Any changes to IDLs or other encodings will be backwards-compatible. | +| transport API | All exposed transport APIs for communication to/from binary components. | All transport APIs (e.g. m3msg, Thrift, HTTP+JSON) will maintain their current surface area, definitions, semantics, and behavior. Any changes to APIs will be additive or done semantically and with wire compatibility. | + +This version does **not** guarantee library-level compatibility. Users are not +encouraged to integrate libraries from m3db/m3 directly: use at your own risk. From 9cb09c869f209e9da6c7c749734822e1dd24902b Mon Sep 17 00:00:00 2001 From: Matt Way Date: Mon, 9 Nov 2020 16:17:30 -0500 Subject: [PATCH 45/47] Add GOVERNANCE.md to describe governance (#2830) --- GOVERNANCE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 GOVERNANCE.md diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000000..b3c7a5cc9f --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,20 @@ +# Governance + +M3 is owned by Uber, and is governed by its Technical Steering Committee (the +"TSC"). + +The current members of the TSC are: + +- @jimjag (Uber) +- @martin-mao (Chronosphere) +- @mway (Uber) +- @prateek (Uber) +- @robskillington (Chronosphere) + +While the TSC aims to operate as a consensus-based community, if any TSC +decision requires a vote to move the proposal forward, decisions by vote require +a majority vote of all TSC members. + +To reach the TSC directly, please email m3-tsc@uber.com. For administrative +questions or issues, please contact ospo@uber.com. To add the TSC to issues or +pull requests, add the [TSC label](https://github.com/m3db/m3/labels/TSC). From 9ba35adea72dafa16b3c75b83387b4909f1147a2 Mon Sep 17 00:00:00 2001 From: Matt Way Date: Mon, 9 Nov 2020 16:20:14 -0500 Subject: [PATCH 46/47] Add coding style guide (#2831) --- STYLEGUIDE.md | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 STYLEGUIDE.md diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md new file mode 100644 index 0000000000..33352c7e13 --- /dev/null +++ b/STYLEGUIDE.md @@ -0,0 +1,168 @@ +# M3 Coding Styleguide + +M3's umbrella coding style guide is Uber's [Go Style Guide][uber-guide]. This +document is maintained as a superset of that guide, capturing any substantive, +intended departures from Uber's coding style. Where possible, code should follow +these guidelines to ensure consistent style and coding practices across the +codebase. + +Above all else, this guide is intended to be a point of reference rather than +a sacred text. We maintain a style guide as a pragmatic tool that can be used to +avoid common issues and normalize code across many authors and the Go community. + +New code should follow the style guide by default, preferring guidelines +established here or in Uber's guide over any conflicting, pre-existing +precedents in the M3 codebase. Ultimately, the hope is that the codebase +incrementally moves closer to the style guide with each change. + +Since the M3 monorepo predated this style guide, reviewers should not expect +contributors to make unrelated or unreasonable style-based changes as part of +pull requests. However, when changing code that could reasonably be updated +to follow the guide, we prefer that those changes adopt the guidelines to avoid +sustaining or increasing technical debt. See DEVELOPMENT.md for more detail on +changes involving style. + +[uber-guide]: https://github.com/uber-go/guide/blob/master/style.md + +## Linting + +Many guidelines are flagged by `go vet` or the other configured linters (see +[.golangci.yml][.golangci.yml]). Wherever possible, we prefer to use tooling to +enforce style to remove subjectivity or ambiguity. Linting is also a blocking +build for merging pull requests. + +[.golangci.yml]: https://github.com/m3db/m3/blob/master/.golangci.yml + +## Template + +When adding to this guide, use the following template: + +~~~ +### Short sentence about the guideline. + +Clearly (and succinctly) articulate the guideline and its rationale, including +any problematic counter-examples. Be intentional when using language like +"always", "never", etc, instead using words like "prefer" and "avoid" if the +guideline isn't a hard rule. If it makes sense, also include example code: + + + + + + +
BadGood
+ +```go +goodExample := false +``` + + + +```go +goodExample := true +``` + +
+Description of bad code. + +Description of good code. +
+~~~ + +## Guidelines + +### Export types carefully. + +Types should only be exported if they must be used by multiple packages. This +applies also to adding new packages: a new package should only be added if it +will be imported by multiple packages. If a given type or package will only +initially be imported in one package, define those type(s) in that importing +package instead. + +In general, it's harder to reduce surface area than it is to incrementally +increase surface area, and the former is a breaking change while the latter is +often not. + +### Treat flaky tests like consistent failures. + +Flaky tests add noise to code health signals, reduce trust in tests to be +representative of code behavior. Worse, flaky tests can be either false positive +or false negative, making it especially unclear as to whether or not a given +test passing or failing is good or bad. All of these reduce overall velocity +and/or reliability. + +All tests discovered to be flaky should be immediately result in either (a) the +test being skipped because it is unreliable, or (b) master being frozen until +the test is fixed and proven to no longer be flaky. + +### Do not expose experimental package types in non-experimental packages. + +A package is only able to guarantee a level of maturity/stability that is the +lowest common denominator of all of its composing or transitively exported +types. Given a hypothetical scenario: + +```go +package foo + +type Bar { + Baz xfoo.Baz +} +``` + +In this case, the stability of `foo.Bar` is purportedly guaranteed by package +`foo` being non-experimental, but since it transitively exposes `xfoo.Baz` as +part of `foo.Bar`, either (a) `xfoo.Baz` must implicitly adhere to versioning +compatibility guarantees or (b) `foo` can no longer be considered stable, +as any breaking change to `xfoo.Baz` will break `foo`. + +This is spiritually similar to the +[Avoid Embedding Types In Public Structs][avoid-embedding-types] guidance, in +that it bleeds implementation and compatibility details in an inappropriate way. + +This guidance also applies to any cases in which `internal` packages are used: +any `internal` type is essentially the same as an unexported type, meaning that +that type is only implicitly available to users. + +[avoid-embedding-types]: https://github.com/uber-go/guide/blob/master/style.md#avoid-embedding-types-in-public-structs + + + + + + +
BadGood
+ +```go +type NewConnectionFn func( + channelName string, addr string, opts Options, +) (xclose.SimpleCloser, rpc.TChanNode, error) +``` + + + +```go +type NewConnectionFn func( + channelName string, addr string, opts Options, +) (io.Closer, rpc.TChanNode, error) + +// or + +type SimpleCloser = func() + +type NewConnectionFn func( + channelName string, addr string, opts Options, +) (SimpleCloser, rpc.TChanNode, error) +``` + +
+ +`xclose.SimpleCloser` is part of `x/close`, an experimental package, but is +directly exposed as part of `src/dbnode/client.NewConnectionFn`. + + + +The canonical `io.Closer` is used instead, or a type alias representing +`xclose.SimpleCloser` is used instead. Both options prevent leaking experimental +packages as part of non-experimental library APIs. + +
From c6a256d42d334a89407bed6e8fc18e9ad81b2379 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 9 Nov 2020 17:19:42 -0500 Subject: [PATCH 47/47] Replace closer with resource package (#2864) --- src/aggregator/aggregator/map.go | 4 +- src/aggregator/runtime/options_manager.go | 6 +- src/dbnode/client/connection_pool.go | 6 +- src/dbnode/client/connection_pool_test.go | 6 +- src/dbnode/client/options.go | 4 +- src/dbnode/client/session.go | 4 +- .../session_fetch_high_concurrency_test.go | 4 +- src/dbnode/digest/fd_digest.go | 4 +- src/dbnode/namespace/namespace_mock.go | 6 +- .../namespace/namespace_runtime_options.go | 6 +- src/dbnode/namespace/schema_registry.go | 4 +- src/dbnode/namespace/types.go | 4 +- .../network/server/httpjson/cluster/server.go | 4 +- .../server/tchannelthrift/cluster/server.go | 4 +- .../server/tchannelthrift/node/service.go | 4 +- src/dbnode/persist/fs/files.go | 12 --- src/dbnode/persist/fs/files_test.go | 3 +- src/dbnode/persist/fs/persist_manager.go | 4 +- src/dbnode/persist/fs/write.go | 3 +- src/dbnode/runtime/runtime_mock.go | 6 +- src/dbnode/runtime/runtime_options_manager.go | 6 +- src/dbnode/runtime/types.go | 4 +- .../bootstrap/bootstrapper/peers/source.go | 4 +- src/dbnode/storage/index.go | 17 ++-- src/dbnode/storage/index/block.go | 22 ++--- src/dbnode/storage/index/block_prop_test.go | 4 +- src/dbnode/storage/index/block_test.go | 84 +++++++++++++------ src/dbnode/storage/index/mutable_segments.go | 4 +- src/dbnode/storage/index/types.go | 6 +- .../storage/index_query_concurrent_test.go | 6 +- src/dbnode/storage/namespace.go | 4 +- src/dbnode/storage/shard.go | 4 +- src/dbnode/x/xio/types.go | 4 +- .../storage/m3/cluster_namespaces_watcher.go | 6 +- src/query/storage/m3/types.go | 4 +- src/x/checked/checked_mock.go | 4 +- src/x/checked/ref.go | 6 +- src/x/checked/ref_test.go | 6 +- src/x/checked/types.go | 4 +- src/x/context/context.go | 10 +-- src/x/context/context_test.go | 20 ++--- src/x/context/pool_test.go | 4 +- src/x/context/types.go | 6 +- src/x/{close => resource}/close.go | 46 ++++------ src/x/{close => resource}/close_test.go | 11 ++- src/x/resource/types.go | 23 +++-- src/x/watch/source.go | 4 +- src/x/watch/watch.go | 6 +- 48 files changed, 223 insertions(+), 204 deletions(-) rename src/x/{close => resource}/close.go (70%) rename src/x/{close => resource}/close_test.go (92%) diff --git a/src/aggregator/aggregator/map.go b/src/aggregator/aggregator/map.go index 130c20d680..6d8139390c 100644 --- a/src/aggregator/aggregator/map.go +++ b/src/aggregator/aggregator/map.go @@ -35,7 +35,7 @@ import ( "github.com/m3db/m3/src/metrics/metric/aggregated" "github.com/m3db/m3/src/metrics/metric/unaggregated" "github.com/m3db/m3/src/x/clock" - "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" "github.com/uber-go/tally" ) @@ -108,7 +108,7 @@ type metricMap struct { firstInsertAt time.Time rateLimiter *rate.Limiter runtimeOpts runtime.Options - runtimeOptsCloser close.SimpleCloser + runtimeOptsCloser xresource.SimpleCloser sleepFn sleepFn metrics metricMapMetrics } diff --git a/src/aggregator/runtime/options_manager.go b/src/aggregator/runtime/options_manager.go index 08c26060f2..b2182d81a0 100644 --- a/src/aggregator/runtime/options_manager.go +++ b/src/aggregator/runtime/options_manager.go @@ -21,7 +21,7 @@ package runtime import ( - "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" "github.com/m3db/m3/src/x/watch" ) @@ -36,7 +36,7 @@ type OptionsManager interface { // RegisterWatcher registers a watcher that watches updates to runtime options. // When an update arrives, the manager will deliver the update to all registered // watchers. - RegisterWatcher(l OptionsWatcher) close.SimpleCloser + RegisterWatcher(l OptionsWatcher) xresource.SimpleCloser // Close closes the watcher and all descendent watches Close() @@ -72,7 +72,7 @@ func (w *optionsManager) RuntimeOptions() Options { func (w *optionsManager) RegisterWatcher( watcher OptionsWatcher, -) close.SimpleCloser { +) xresource.SimpleCloser { _, watch, _ := w.watchable.Watch() // The watchable is always initialized so it's okay to do a blocking read. diff --git a/src/dbnode/client/connection_pool.go b/src/dbnode/client/connection_pool.go index e8db551890..4d1e1af1d2 100644 --- a/src/dbnode/client/connection_pool.go +++ b/src/dbnode/client/connection_pool.go @@ -31,7 +31,7 @@ import ( "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/topology" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" murmur3 "github.com/m3db/stackmurmur3/v2" "github.com/uber-go/tally" @@ -68,14 +68,14 @@ type connPool struct { } type conn struct { - channel xclose.SimpleCloser + channel xresource.SimpleCloser client rpc.TChanNode } // NewConnectionFn is a function that creates a connection. type NewConnectionFn func( channelName string, addr string, opts Options, -) (xclose.SimpleCloser, rpc.TChanNode, error) +) (xresource.SimpleCloser, rpc.TChanNode, error) type healthCheckFn func(client rpc.TChanNode, opts Options) error diff --git a/src/dbnode/client/connection_pool_test.go b/src/dbnode/client/connection_pool_test.go index 0724171f6f..f4d391180c 100644 --- a/src/dbnode/client/connection_pool_test.go +++ b/src/dbnode/client/connection_pool_test.go @@ -30,7 +30,7 @@ import ( "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/topology" xclock "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" "github.com/stretchr/testify/require" "github.com/golang/mock/gomock" @@ -85,7 +85,7 @@ func TestConnectionPoolConnectsAndRetriesConnects(t *testing.T) { fn := func( ch string, addr string, opts Options, - ) (xclose.SimpleCloser, rpc.TChanNode, error) { + ) (xresource.SimpleCloser, rpc.TChanNode, error) { attempt := int(atomic.AddInt32(&attempts, 1)) if attempt == 1 { return nil, nil, fmt.Errorf("a connect error") @@ -237,7 +237,7 @@ func TestConnectionPoolHealthChecks(t *testing.T) { fn := func( ch string, addr string, opts Options, - ) (xclose.SimpleCloser, rpc.TChanNode, error) { + ) (xresource.SimpleCloser, rpc.TChanNode, error) { attempt := atomic.AddInt32(&newConnAttempt, 1) if attempt == 1 { return channelNone, client1, nil diff --git a/src/dbnode/client/options.go b/src/dbnode/client/options.go index e36001ff83..c8e350b199 100644 --- a/src/dbnode/client/options.go +++ b/src/dbnode/client/options.go @@ -38,11 +38,11 @@ import ( "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" + xresource "github.com/m3db/m3/src/x/resource" xretry "github.com/m3db/m3/src/x/retry" "github.com/m3db/m3/src/x/sampler" "github.com/m3db/m3/src/x/serialize" @@ -319,7 +319,7 @@ func NewOptionsForAsyncClusters(opts Options, topoInits []topology.Initializer, func defaultNewConnectionFn( channelName string, address string, opts Options, -) (xclose.SimpleCloser, rpc.TChanNode, error) { +) (xresource.SimpleCloser, rpc.TChanNode, error) { channel, err := tchannel.NewChannel(channelName, opts.ChannelOptions()) if err != nil { return nil, nil, err diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index f803750d0b..a188f105cf 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -48,12 +48,12 @@ import ( "github.com/m3db/m3/src/dbnode/x/xpool" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" + xresource "github.com/m3db/m3/src/x/resource" xretry "github.com/m3db/m3/src/x/retry" "github.com/m3db/m3/src/x/sampler" "github.com/m3db/m3/src/x/serialize" @@ -136,7 +136,7 @@ type sessionState struct { type session struct { state sessionState opts Options - runtimeOptsListenerCloser xclose.Closer + runtimeOptsListenerCloser xresource.SimpleCloser scope tally.Scope nowFn clock.NowFn log *zap.Logger diff --git a/src/dbnode/client/session_fetch_high_concurrency_test.go b/src/dbnode/client/session_fetch_high_concurrency_test.go index 77a07ea747..a4acd087c3 100644 --- a/src/dbnode/client/session_fetch_high_concurrency_test.go +++ b/src/dbnode/client/session_fetch_high_concurrency_test.go @@ -34,8 +34,8 @@ import ( "github.com/m3db/m3/src/dbnode/sharding" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/ts" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/ident" + xresource "github.com/m3db/m3/src/x/resource" xtime "github.com/m3db/m3/src/x/time" "github.com/golang/mock/gomock" @@ -102,7 +102,7 @@ func TestSessionFetchIDsHighConcurrency(t *testing.T) { // to be able to mock the entire end to end pipeline newConnFn := func( _ string, addr string, _ Options, - ) (xclose.SimpleCloser, rpc.TChanNode, error) { + ) (xresource.SimpleCloser, rpc.TChanNode, error) { mockClient := rpc.NewMockTChanNode(ctrl) mockClient.EXPECT().Health(gomock.Any()). Return(healthCheckResult, nil). diff --git a/src/dbnode/digest/fd_digest.go b/src/dbnode/digest/fd_digest.go index af316ff3e3..c8db06a14c 100644 --- a/src/dbnode/digest/fd_digest.go +++ b/src/dbnode/digest/fd_digest.go @@ -25,12 +25,12 @@ import ( "hash/adler32" "os" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" ) // FdWithDigest is a container for a file descriptor and the digest for the file contents. type FdWithDigest interface { - xclose.Closer + xresource.Closer // Fd returns the file descriptor. Fd() *os.File diff --git a/src/dbnode/namespace/namespace_mock.go b/src/dbnode/namespace/namespace_mock.go index 1807675287..4fba8bd53d 100644 --- a/src/dbnode/namespace/namespace_mock.go +++ b/src/dbnode/namespace/namespace_mock.go @@ -30,9 +30,9 @@ import ( "github.com/m3db/m3/src/cluster/client" "github.com/m3db/m3/src/dbnode/retention" - "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" + "github.com/m3db/m3/src/x/resource" "github.com/gogo/protobuf/proto" "github.com/golang/mock/gomock" @@ -879,10 +879,10 @@ func (mr *MockSchemaRegistryMockRecorder) SetSchemaHistory(id, history interface } // RegisterListener mocks base method -func (m *MockSchemaRegistry) RegisterListener(id ident.ID, listener SchemaListener) (close.SimpleCloser, error) { +func (m *MockSchemaRegistry) RegisterListener(id ident.ID, listener SchemaListener) (resource.SimpleCloser, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegisterListener", id, listener) - ret0, _ := ret[0].(close.SimpleCloser) + ret0, _ := ret[0].(resource.SimpleCloser) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/src/dbnode/namespace/namespace_runtime_options.go b/src/dbnode/namespace/namespace_runtime_options.go index 8531cc6eaa..26538c7a7a 100644 --- a/src/dbnode/namespace/namespace_runtime_options.go +++ b/src/dbnode/namespace/namespace_runtime_options.go @@ -23,7 +23,7 @@ package namespace import ( "sync" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" "github.com/m3db/m3/src/x/watch" ) @@ -88,7 +88,7 @@ type RuntimeOptionsManager interface { // RegisterListener registers a listener for updates to runtime options, // it will synchronously call back the listener when this method is called // to deliver the current set of runtime options. - RegisterListener(l RuntimeOptionsListener) xclose.SimpleCloser + RegisterListener(l RuntimeOptionsListener) xresource.SimpleCloser // Close closes the watcher and all descendent watches. Close() @@ -228,7 +228,7 @@ func (w *runtimeOptionsManager) Get() RuntimeOptions { func (w *runtimeOptionsManager) RegisterListener( listener RuntimeOptionsListener, -) xclose.SimpleCloser { +) xresource.SimpleCloser { _, watch, _ := w.watchable.Watch() // We always initialize the watchable so always read diff --git a/src/dbnode/namespace/schema_registry.go b/src/dbnode/namespace/schema_registry.go index 014f19d4d2..08e8829cbb 100644 --- a/src/dbnode/namespace/schema_registry.go +++ b/src/dbnode/namespace/schema_registry.go @@ -24,8 +24,8 @@ import ( "fmt" "sync" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/ident" + xresource "github.com/m3db/m3/src/x/resource" xwatch "github.com/m3db/m3/src/x/watch" "go.uber.org/zap" @@ -145,7 +145,7 @@ func (sr *schemaRegistry) getSchemaHistory(nsIDStr string) (SchemaHistory, error func (sr *schemaRegistry) RegisterListener( nsID ident.ID, listener SchemaListener, -) (xclose.SimpleCloser, error) { +) (xresource.SimpleCloser, error) { if !sr.protoEnabled { return nil, nil } diff --git a/src/dbnode/namespace/types.go b/src/dbnode/namespace/types.go index 39b71a1154..8afe60ea8d 100644 --- a/src/dbnode/namespace/types.go +++ b/src/dbnode/namespace/types.go @@ -25,9 +25,9 @@ import ( "github.com/m3db/m3/src/cluster/client" "github.com/m3db/m3/src/dbnode/retention" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" "github.com/gogo/protobuf/proto" ) @@ -205,7 +205,7 @@ type SchemaRegistry interface { // RegisterListener registers a schema listener for the namespace. // If proto is not enabled, nil, nil is returned - RegisterListener(id ident.ID, listener SchemaListener) (xclose.SimpleCloser, error) + RegisterListener(id ident.ID, listener SchemaListener) (xresource.SimpleCloser, error) // Close closes all the listeners. Close() diff --git a/src/dbnode/network/server/httpjson/cluster/server.go b/src/dbnode/network/server/httpjson/cluster/server.go index 2ccd905580..70d62a9e43 100644 --- a/src/dbnode/network/server/httpjson/cluster/server.go +++ b/src/dbnode/network/server/httpjson/cluster/server.go @@ -28,8 +28,8 @@ import ( ns "github.com/m3db/m3/src/dbnode/network/server" "github.com/m3db/m3/src/dbnode/network/server/httpjson" ttcluster "github.com/m3db/m3/src/dbnode/network/server/tchannelthrift/cluster" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" + xresource "github.com/m3db/m3/src/x/resource" ) type server struct { @@ -83,6 +83,6 @@ func (s *server) ListenAndServe() (ns.Close, error) { return func() { listener.Close() - xclose.TryClose(service) + xresource.TryClose(service) // nolint: errcheck }, nil } diff --git a/src/dbnode/network/server/tchannelthrift/cluster/server.go b/src/dbnode/network/server/tchannelthrift/cluster/server.go index 0790618a05..5b05af00e8 100644 --- a/src/dbnode/network/server/tchannelthrift/cluster/server.go +++ b/src/dbnode/network/server/tchannelthrift/cluster/server.go @@ -25,8 +25,8 @@ import ( "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" ns "github.com/m3db/m3/src/dbnode/network/server" "github.com/m3db/m3/src/dbnode/network/server/tchannelthrift" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" + xresource "github.com/m3db/m3/src/x/resource" "github.com/uber/tchannel-go" ) @@ -76,6 +76,6 @@ func (s *server) ListenAndServe() (ns.Close, error) { return func() { channel.Close() - xclose.TryClose(service) + xresource.TryClose(service) // nolint: errcheck }, nil } diff --git a/src/dbnode/network/server/tchannelthrift/node/service.go b/src/dbnode/network/server/tchannelthrift/node/service.go index 99d6c9cc51..871dd91b20 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service.go +++ b/src/dbnode/network/server/tchannelthrift/node/service.go @@ -49,7 +49,7 @@ import ( "github.com/m3db/m3/src/x/instrument" xopentracing "github.com/m3db/m3/src/x/opentracing" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" "github.com/m3db/m3/src/x/serialize" xtime "github.com/m3db/m3/src/x/time" @@ -2302,7 +2302,7 @@ func (s *service) readEncodedResult( segments := s.pools.segmentsArray.Get() segments = segmentsArr(segments).grow(len(encoded)) segments = segments[:0] - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { s.pools.segmentsArray.Put(segments) })) diff --git a/src/dbnode/persist/fs/files.go b/src/dbnode/persist/fs/files.go index 1de9a425d6..a0017e8075 100644 --- a/src/dbnode/persist/fs/files.go +++ b/src/dbnode/persist/fs/files.go @@ -38,7 +38,6 @@ import ( "github.com/m3db/m3/src/dbnode/persist/fs/msgpack" "github.com/m3db/m3/src/dbnode/persist/schema" idxpersist "github.com/m3db/m3/src/m3ninx/persist" - xclose "github.com/m3db/m3/src/x/close" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" @@ -360,17 +359,6 @@ func openFiles(opener fileOpener, fds map[string]**os.File) error { return firstErr } -// TODO(xichen): move closeAll to m3x/close. -func closeAll(closers ...xclose.Closer) error { - multiErr := xerrors.NewMultiError() - for _, closer := range closers { - if err := closer.Close(); err != nil { - multiErr = multiErr.Add(err) - } - } - return multiErr.FinalError() -} - // DeleteFiles delete a set of files, returning all the errors encountered during // the deletion process. func DeleteFiles(filePaths []string) error { diff --git a/src/dbnode/persist/fs/files_test.go b/src/dbnode/persist/fs/files_test.go index 60a4b68dd0..556bf9efb2 100644 --- a/src/dbnode/persist/fs/files_test.go +++ b/src/dbnode/persist/fs/files_test.go @@ -39,6 +39,7 @@ import ( "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" "github.com/pborman/uuid" "github.com/stretchr/testify/assert" @@ -88,7 +89,7 @@ func TestCloseAllFails(t *testing.T) { defer os.Remove(file.Name()) assert.NoError(t, file.Close()) - assert.Error(t, closeAll(file)) + assert.Error(t, xresource.CloseAll(file)) } func TestDeleteFiles(t *testing.T) { diff --git a/src/dbnode/persist/fs/persist_manager.go b/src/dbnode/persist/fs/persist_manager.go index 37e1021b8c..8d201a298b 100644 --- a/src/dbnode/persist/fs/persist_manager.go +++ b/src/dbnode/persist/fs/persist_manager.go @@ -35,8 +35,8 @@ import ( m3ninxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" "github.com/pborman/uuid" "github.com/uber-go/tally" @@ -93,7 +93,7 @@ type persistManager struct { metrics persistManagerMetrics - runtimeOptsListener xclose.SimpleCloser + runtimeOptsListener xresource.SimpleCloser } type dataPersistManager struct { diff --git a/src/dbnode/persist/fs/write.go b/src/dbnode/persist/fs/write.go index 5b2c17e6a6..939609547c 100644 --- a/src/dbnode/persist/fs/write.go +++ b/src/dbnode/persist/fs/write.go @@ -36,6 +36,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/ident" + xresource "github.com/m3db/m3/src/x/resource" "github.com/m3db/m3/src/x/serialize" xtime "github.com/m3db/m3/src/x/time" @@ -390,7 +391,7 @@ func (w *writer) closeWOIndex() error { return err } - return closeAll( + return xresource.CloseAll( w.infoFdWithDigest, w.indexFdWithDigest, w.summariesFdWithDigest, diff --git a/src/dbnode/runtime/runtime_mock.go b/src/dbnode/runtime/runtime_mock.go index aa45644751..187dc92e50 100644 --- a/src/dbnode/runtime/runtime_mock.go +++ b/src/dbnode/runtime/runtime_mock.go @@ -30,7 +30,7 @@ import ( "github.com/m3db/m3/src/dbnode/ratelimit" "github.com/m3db/m3/src/dbnode/topology" - "github.com/m3db/m3/src/x/close" + "github.com/m3db/m3/src/x/resource" "github.com/golang/mock/gomock" ) @@ -488,10 +488,10 @@ func (mr *MockOptionsManagerMockRecorder) Get() *gomock.Call { } // RegisterListener mocks base method -func (m *MockOptionsManager) RegisterListener(l OptionsListener) close.SimpleCloser { +func (m *MockOptionsManager) RegisterListener(l OptionsListener) resource.SimpleCloser { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegisterListener", l) - ret0, _ := ret[0].(close.SimpleCloser) + ret0, _ := ret[0].(resource.SimpleCloser) return ret0 } diff --git a/src/dbnode/runtime/runtime_options_manager.go b/src/dbnode/runtime/runtime_options_manager.go index 290b6ca7b0..bcb8630b13 100644 --- a/src/dbnode/runtime/runtime_options_manager.go +++ b/src/dbnode/runtime/runtime_options_manager.go @@ -23,7 +23,7 @@ package runtime import ( "fmt" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" xwatch "github.com/m3db/m3/src/x/watch" ) @@ -52,7 +52,7 @@ func (w *optionsManager) Get() Options { func (w *optionsManager) RegisterListener( listener OptionsListener, -) xclose.SimpleCloser { +) xresource.SimpleCloser { _, watch, _ := w.watchable.Watch() // We always initialize the watchable so always read @@ -98,7 +98,7 @@ func (n noOpOptionsManager) Get() Options { func (n noOpOptionsManager) RegisterListener( listener OptionsListener, -) xclose.SimpleCloser { +) xresource.SimpleCloser { // noOpOptionsManager never changes its options, not worth // registering listener return noOpCloser{} diff --git a/src/dbnode/runtime/types.go b/src/dbnode/runtime/types.go index cafe9f8046..4b8ae81f9a 100644 --- a/src/dbnode/runtime/types.go +++ b/src/dbnode/runtime/types.go @@ -25,7 +25,7 @@ import ( "github.com/m3db/m3/src/dbnode/ratelimit" "github.com/m3db/m3/src/dbnode/topology" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" ) // Options is a set of runtime options. @@ -195,7 +195,7 @@ type OptionsManager interface { // RegisterListener registers a listener for updates to runtime options, // it will synchronously call back the listener when this method is called // to deliver the current set of runtime options. - RegisterListener(l OptionsListener) xclose.SimpleCloser + RegisterListener(l OptionsListener) xresource.SimpleCloser // Close closes the watcher and all descendent watches. Close() diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go index 3f09a360ef..d4f7e05560 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go @@ -45,10 +45,10 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/fst" idxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" xsync "github.com/m3db/m3/src/x/sync" xtime "github.com/m3db/m3/src/x/time" @@ -340,7 +340,7 @@ func (s *peersSource) startPersistenceQueueWorkerLoop( persistFlush, bootstrapResult, lock) }() - return xclose.CloserFn(persistFlush.DoneFlush), nil + return xresource.CloserFn(persistFlush.DoneFlush), nil } // runPersistenceQueueWorkerLoop is meant to be run in its own goroutine, and it creates a worker that diff --git a/src/dbnode/storage/index.go b/src/dbnode/storage/index.go index 9dabe60f96..6a61795e8b 100644 --- a/src/dbnode/storage/index.go +++ b/src/dbnode/storage/index.go @@ -53,13 +53,12 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment/builder" idxpersist "github.com/m3db/m3/src/m3ninx/persist" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" xopentracing "github.com/m3db/m3/src/x/opentracing" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" xsync "github.com/m3db/m3/src/x/sync" xtime "github.com/m3db/m3/src/x/time" @@ -117,8 +116,8 @@ type nsIndex struct { logger *zap.Logger opts Options nsMetadata namespace.Metadata - runtimeOptsListener xclose.SimpleCloser - runtimeNsOptsListener xclose.SimpleCloser + runtimeOptsListener xresource.SimpleCloser + runtimeNsOptsListener xresource.SimpleCloser resultsPool index.QueryResultsPool aggregateResultsPool index.AggregateResultsPool @@ -211,7 +210,7 @@ type newNamespaceIndexOpts struct { // execBlockQueryFn executes a query against the given block whilst tracking state. type execBlockQueryFn func( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, block index.Block, query index.Query, opts index.QueryOptions, @@ -1555,7 +1554,7 @@ func (i *nsIndex) queryWithSpan( // Create a cancellable lifetime and cancel it at end of this method so that // no child async task modifies the result after this method returns. - cancellable := resource.NewCancellableLifetime() + cancellable := xresource.NewCancellableLifetime() defer cancellable.Cancel() for _, block := range blocks { @@ -1663,7 +1662,7 @@ func (i *nsIndex) queryWithSpan( func (i *nsIndex) execBlockQueryFn( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, block index.Block, query index.Query, opts index.QueryOptions, @@ -1701,7 +1700,7 @@ func (i *nsIndex) execBlockQueryFn( func (i *nsIndex) execBlockWideQueryFn( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, block index.Block, query index.Query, opts index.QueryOptions, @@ -1745,7 +1744,7 @@ func (i *nsIndex) execBlockWideQueryFn( func (i *nsIndex) execBlockAggregateQueryFn( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, block index.Block, query index.Query, opts index.QueryOptions, diff --git a/src/dbnode/storage/index/block.go b/src/dbnode/storage/index/block.go index f1f0e80b9f..120fe09a3d 100644 --- a/src/dbnode/storage/index/block.go +++ b/src/dbnode/storage/index/block.go @@ -43,7 +43,7 @@ import ( xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" xtime "github.com/m3db/m3/src/x/time" "github.com/opentracing/opentracing-go" @@ -400,7 +400,7 @@ func (b *block) segmentReadersWithRLock() ([]segment.Reader, error) { // to the results datastructure). func (b *block) Query( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, query Query, opts QueryOptions, results BaseResults, @@ -420,7 +420,7 @@ func (b *block) Query( func (b *block) queryWithSpan( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, query Query, opts QueryOptions, results BaseResults, @@ -465,7 +465,7 @@ func (b *block) queryWithSpan( return false, errCancelledQuery } execCloseRegistered = true // Make sure to not locally close it. - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { b.closeAsync(exec) })) cancellable.ReleaseCheckout() @@ -530,7 +530,7 @@ func (b *block) closeAsync(closer io.Closer) { } func (b *block) addQueryResults( - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, results BaseResults, batch []doc.Document, ) ([]doc.Document, int, int, error) { @@ -548,7 +548,7 @@ func (b *block) addQueryResults( return batch, 0, 0, errCancelledQuery } - // try to add the docs to the resource. + // try to add the docs to the xresource. size, docsCount, err := results.AddDocuments(batch) // immediately release the checkout on the lifetime of query. @@ -572,7 +572,7 @@ func (b *block) addQueryResults( // pre-aggregated results via the FST underlying the index. func (b *block) Aggregate( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, opts QueryOptions, results AggregateResults, logFields []opentracinglog.Field, @@ -591,7 +591,7 @@ func (b *block) Aggregate( func (b *block) aggregateWithSpan( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, opts QueryOptions, results AggregateResults, sp opentracing.Span, @@ -667,7 +667,7 @@ func (b *block) aggregateWithSpan( // read by the readers. for _, reader := range readers { reader := reader // Capture for inline function. - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { b.closeAsync(reader) })) } @@ -792,7 +792,7 @@ func (b *block) pooledID(id []byte) ident.ID { } func (b *block) addAggregateResults( - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, results AggregateResults, batch []AggregateResultsEntry, ) ([]AggregateResultsEntry, int, int, error) { @@ -810,7 +810,7 @@ func (b *block) addAggregateResults( return batch, 0, 0, errCancelledQuery } - // try to add the docs to the resource. + // try to add the docs to the xresource. size, docsCount := results.AddFields(batch) // immediately release the checkout on the lifetime of query. diff --git a/src/dbnode/storage/index/block_prop_test.go b/src/dbnode/storage/index/block_prop_test.go index 447ca2f12e..0624a3923a 100644 --- a/src/dbnode/storage/index/block_prop_test.go +++ b/src/dbnode/storage/index/block_prop_test.go @@ -40,7 +40,7 @@ import ( "github.com/m3db/m3/src/m3ninx/search/proptest" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/instrument" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" "github.com/leanovate/gopter" "github.com/leanovate/gopter/prop" @@ -107,7 +107,7 @@ func TestPostingsListCacheDoesNotAffectBlockQueryResults(t *testing.T) { idx.NewQueryFromSearchQuery(q), } - cancellable := resource.NewCancellableLifetime() + cancellable := xresource.NewCancellableLifetime() cancelled := false doneQuery := func() { if !cancelled { diff --git a/src/dbnode/storage/index/block_test.go b/src/dbnode/storage/index/block_test.go index 5bb078b676..e8f4e64ee4 100644 --- a/src/dbnode/storage/index/block_test.go +++ b/src/dbnode/storage/index/block_test.go @@ -40,7 +40,7 @@ import ( "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" xtime "github.com/m3db/m3/src/x/time" "github.com/golang/mock/gomock" @@ -366,7 +366,7 @@ func TestBlockQueryAfterClose(t *testing.T) { require.Equal(t, start.Add(time.Hour), b.EndTime()) require.NoError(t, b.Close()) - _, err = b.Query(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Query(context.NewContext(), xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, nil, emptyLogFields) require.Error(t, err) } @@ -382,7 +382,7 @@ func TestBlockQueryWithCancelledQuery(t *testing.T) { require.Equal(t, start.Add(time.Hour), b.EndTime()) // Precancel query. - cancellable := resource.NewCancellableLifetime() + cancellable := xresource.NewCancellableLifetime() cancellable.Cancel() _, err = b.Query(context.NewContext(), cancellable, @@ -405,7 +405,7 @@ func TestBlockQueryExecutorError(t *testing.T) { return nil, fmt.Errorf("random-err") } - _, err = b.Query(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Query(context.NewContext(), xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, nil, emptyLogFields) require.Error(t, err) } @@ -428,7 +428,7 @@ func TestBlockQuerySegmentReaderError(t *testing.T) { randErr := fmt.Errorf("random-err") seg.EXPECT().Reader().Return(nil, randErr) - _, err = b.Query(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Query(context.NewContext(), xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, nil, emptyLogFields) require.Equal(t, randErr, err) } @@ -468,7 +468,7 @@ func TestBlockQueryAddResultsSegmentsError(t *testing.T) { randErr := fmt.Errorf("random-err") seg3.EXPECT().Reader().Return(nil, randErr) - _, err = b.Query(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Query(context.NewContext(), xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, nil, emptyLogFields) require.Equal(t, randErr, err) } @@ -495,7 +495,7 @@ func TestBlockMockQueryExecutorExecError(t *testing.T) { exec.EXPECT().Execute(gomock.Any()).Return(nil, fmt.Errorf("randomerr")), exec.EXPECT().Close(), ) - _, err = b.Query(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Query(context.NewContext(), xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, nil, emptyLogFields) require.Error(t, err) } @@ -531,7 +531,7 @@ func TestBlockMockQueryExecutorExecIterErr(t *testing.T) { ctx := context.NewContext() - _, err = b.Query(ctx, resource.NewCancellableLifetime(), + _, err = b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, NewQueryResults(nil, QueryResultsOptions{}, testOpts), emptyLogFields) require.Error(t, err) @@ -575,7 +575,7 @@ func TestBlockMockQueryExecutorExecLimit(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{SeriesLimit: limit}, results, emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -622,7 +622,7 @@ func TestBlockMockQueryExecutorExecIterCloseErr(t *testing.T) { ctx := context.NewContext() - _, err = b.Query(ctx, resource.NewCancellableLifetime(), + _, err = b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, results, emptyLogFields) require.Error(t, err) @@ -664,7 +664,7 @@ func TestBlockMockQuerySeriesLimitNonExhaustive(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{SeriesLimit: limit}, results, emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -715,7 +715,7 @@ func TestBlockMockQuerySeriesLimitExhaustive(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{SeriesLimit: limit}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -766,7 +766,7 @@ func TestBlockMockQueryDocsLimitNonExhaustive(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{DocsLimit: docsLimit}, results, emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -817,7 +817,7 @@ func TestBlockMockQueryDocsLimitExhaustive(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{DocsLimit: docsLimit}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -871,7 +871,7 @@ func TestBlockMockQueryMergeResultsMapLimit(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{SeriesLimit: limit}, results, emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -926,7 +926,7 @@ func TestBlockMockQueryMergeResultsDupeID(t *testing.T) { ctx := context.NewContext() - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), defaultQuery, QueryOptions{}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -1395,7 +1395,7 @@ func TestBlockE2EInsertQuery(t *testing.T) { ctx.SetGoContext(opentracing.ContextWithSpan(stdlibctx.Background(), sp)) results := NewQueryResults(nil, QueryResultsOptions{}, testOpts) - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), Query{q}, QueryOptions{}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -1471,7 +1471,7 @@ func TestBlockE2EInsertQueryLimit(t *testing.T) { limit := 1 results := NewQueryResults(nil, QueryResultsOptions{SizeLimit: limit}, testOpts) - exhaustive, err := b.Query(context.NewContext(), resource.NewCancellableLifetime(), + exhaustive, err := b.Query(context.NewContext(), xresource.NewCancellableLifetime(), Query{q}, QueryOptions{SeriesLimit: limit}, results, emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -1560,7 +1560,7 @@ func TestBlockE2EInsertAddResultsQuery(t *testing.T) { ctx.SetGoContext(opentracing.ContextWithSpan(stdlibctx.Background(), sp)) results := NewQueryResults(nil, QueryResultsOptions{}, testOpts) - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), Query{q}, QueryOptions{}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -1639,7 +1639,7 @@ func TestBlockE2EInsertAddResultsMergeQuery(t *testing.T) { ctx.SetGoContext(opentracing.ContextWithSpan(stdlibctx.Background(), sp)) results := NewQueryResults(nil, QueryResultsOptions{}, testOpts) - exhaustive, err := b.Query(ctx, resource.NewCancellableLifetime(), + exhaustive, err := b.Query(ctx, xresource.NewCancellableLifetime(), Query{q}, QueryOptions{}, results, emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -1782,7 +1782,7 @@ func TestBlockAggregateAfterClose(t *testing.T) { require.Equal(t, start.Add(time.Hour), b.EndTime()) require.NoError(t, b.Close()) - _, err = b.Aggregate(context.NewContext(), resource.NewCancellableLifetime(), + _, err = b.Aggregate(context.NewContext(), xresource.NewCancellableLifetime(), QueryOptions{}, nil, emptyLogFields) require.Error(t, err) } @@ -1829,7 +1829,12 @@ func TestBlockAggregateIterationErr(t *testing.T) { ctx := context.NewContext() defer ctx.BlockingClose() - _, err = b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 3}, results, emptyLogFields) + _, err = b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 3}, + results, + emptyLogFields) require.Error(t, err) } @@ -1885,7 +1890,12 @@ func TestBlockAggregate(t *testing.T) { iter.EXPECT().Err().Return(nil), iter.EXPECT().Close().Return(nil), ) - exhaustive, err := b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 3}, results, emptyLogFields) + exhaustive, err := b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 3}, + results, + emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) @@ -1956,7 +1966,12 @@ func TestBlockAggregateNotExhaustive(t *testing.T) { iter.EXPECT().Err().Return(nil), iter.EXPECT().Close().Return(nil), ) - exhaustive, err := b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 1}, results, emptyLogFields) + exhaustive, err := b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 1}, + results, + emptyLogFields) require.NoError(t, err) require.False(t, exhaustive) @@ -2043,7 +2058,12 @@ func TestBlockE2EInsertAggregate(t *testing.T) { sp := mtr.StartSpan("root") ctx.SetGoContext(opentracing.ContextWithSpan(stdlibctx.Background(), sp)) - exhaustive, err := b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 10}, results, emptyLogFields) + exhaustive, err := b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 10}, + results, + emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ @@ -2056,7 +2076,12 @@ func TestBlockE2EInsertAggregate(t *testing.T) { Type: AggregateTagNamesAndValues, FieldFilter: AggregateFieldFilter{[]byte("bar")}, }, testOpts) - exhaustive, err = b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 10}, results, emptyLogFields) + exhaustive, err = b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 10}, + results, + emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{ @@ -2068,7 +2093,12 @@ func TestBlockE2EInsertAggregate(t *testing.T) { Type: AggregateTagNamesAndValues, FieldFilter: AggregateFieldFilter{[]byte("random")}, }, testOpts) - exhaustive, err = b.Aggregate(ctx, resource.NewCancellableLifetime(), QueryOptions{SeriesLimit: 10}, results, emptyLogFields) + exhaustive, err = b.Aggregate( + ctx, + xresource.NewCancellableLifetime(), + QueryOptions{SeriesLimit: 10}, + results, + emptyLogFields) require.NoError(t, err) require.True(t, exhaustive) assertAggregateResultsMapEquals(t, map[string][]string{}, results) diff --git a/src/dbnode/storage/index/mutable_segments.go b/src/dbnode/storage/index/mutable_segments.go index baa904cd7b..256bb4a313 100644 --- a/src/dbnode/storage/index/mutable_segments.go +++ b/src/dbnode/storage/index/mutable_segments.go @@ -35,10 +35,10 @@ import ( "github.com/m3db/m3/src/m3ninx/index/segment" "github.com/m3db/m3/src/m3ninx/index/segment/builder" "github.com/m3db/m3/src/m3ninx/index/segment/fst" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/mmap" + xresource "github.com/m3db/m3/src/x/resource" "github.com/uber-go/tally" "go.uber.org/zap" @@ -73,7 +73,7 @@ type mutableSegments struct { blockOpts BlockOptions opts Options iopts instrument.Options - optsListener xclose.SimpleCloser + optsListener xresource.SimpleCloser writeIndexingConcurrency int metrics mutableSegmentsMetrics diff --git a/src/dbnode/storage/index/types.go b/src/dbnode/storage/index/types.go index 58afff9776..78b2cbc1ea 100644 --- a/src/dbnode/storage/index/types.go +++ b/src/dbnode/storage/index/types.go @@ -41,7 +41,7 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/mmap" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" xtime "github.com/m3db/m3/src/x/time" opentracinglog "github.com/opentracing/opentracing-go/log" @@ -352,7 +352,7 @@ type Block interface { // Query resolves the given query into known IDs. Query( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, query Query, opts QueryOptions, results BaseResults, @@ -364,7 +364,7 @@ type Block interface { // avoid going to documents, relying purely on the indexed FSTs. Aggregate( ctx context.Context, - cancellable *resource.CancellableLifetime, + cancellable *xresource.CancellableLifetime, opts QueryOptions, results AggregateResults, logFields []opentracinglog.Field, diff --git a/src/dbnode/storage/index_query_concurrent_test.go b/src/dbnode/storage/index_query_concurrent_test.go index f7f1172367..0c41e5df02 100644 --- a/src/dbnode/storage/index_query_concurrent_test.go +++ b/src/dbnode/storage/index_query_concurrent_test.go @@ -34,7 +34,7 @@ import ( "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/idx" "github.com/m3db/m3/src/x/context" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" xsync "github.com/m3db/m3/src/x/sync" xtest "github.com/m3db/m3/src/x/test" "go.uber.org/zap" @@ -232,7 +232,7 @@ func testNamespaceIndexHighConcurrentQueries( Query(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func( _ context.Context, - _ *resource.CancellableLifetime, + _ *xresource.CancellableLifetime, _ index.Query, _ index.QueryOptions, _ index.QueryResults, @@ -246,7 +246,7 @@ func testNamespaceIndexHighConcurrentQueries( Query(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func( ctx context.Context, - c *resource.CancellableLifetime, + c *xresource.CancellableLifetime, q index.Query, opts index.QueryOptions, r index.QueryResults, diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index e535169c8c..5367b17574 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -43,12 +43,12 @@ import ( "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" xopentracing "github.com/m3db/m3/src/x/opentracing" + xresource "github.com/m3db/m3/src/x/resource" xsync "github.com/m3db/m3/src/x/sync" xtime "github.com/m3db/m3/src/x/time" @@ -122,7 +122,7 @@ type dbNamespace struct { // schemaDescr caches the latest schema for the namespace. // schemaDescr is updated whenever schema registry is updated. - schemaListener xclose.SimpleCloser + schemaListener xresource.SimpleCloser schemaDescr namespace.SchemaDescr // Contains an entry to all shards for fast shard lookup, an diff --git a/src/dbnode/storage/shard.go b/src/dbnode/storage/shard.go index 056afa8166..49f8642757 100644 --- a/src/dbnode/storage/shard.go +++ b/src/dbnode/storage/shard.go @@ -54,11 +54,11 @@ import ( "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/clock" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/context" xerrors "github.com/m3db/m3/src/x/errors" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" xtime "github.com/m3db/m3/src/x/time" "github.com/gogo/protobuf/proto" @@ -185,7 +185,7 @@ type dbShard struct { contextPool context.Pool flushState shardFlushState tickWg *sync.WaitGroup - runtimeOptsListenClosers []xclose.SimpleCloser + runtimeOptsListenClosers []xresource.SimpleCloser currRuntimeOptions dbShardRuntimeOptions logger *zap.Logger metrics dbShardMetrics diff --git a/src/dbnode/x/xio/types.go b/src/dbnode/x/xio/types.go index 8fe7cf1680..fd02692fed 100644 --- a/src/dbnode/x/xio/types.go +++ b/src/dbnode/x/xio/types.go @@ -26,7 +26,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" ) // BlockReader represents a block reader backed by a @@ -43,7 +43,7 @@ var EmptyBlockReader = BlockReader{} // SegmentReader implements the io reader interface backed by a segment. type SegmentReader interface { io.Reader - resource.Finalizer + xresource.Finalizer // Segment gets the segment read by this reader. Segment() (ts.Segment, error) diff --git a/src/query/storage/m3/cluster_namespaces_watcher.go b/src/query/storage/m3/cluster_namespaces_watcher.go index 945dbd7166..a630d8f45c 100644 --- a/src/query/storage/m3/cluster_namespaces_watcher.go +++ b/src/query/storage/m3/cluster_namespaces_watcher.go @@ -21,7 +21,7 @@ package m3 import ( - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" xwatch "github.com/m3db/m3/src/x/watch" ) @@ -48,7 +48,9 @@ func (n *clusterNamespacesWatcher) Get() ClusterNamespaces { return value.(ClusterNamespaces) } -func (n *clusterNamespacesWatcher) RegisterListener(listener ClusterNamespacesListener) xclose.SimpleCloser { +func (n *clusterNamespacesWatcher) RegisterListener( + listener ClusterNamespacesListener, +) xresource.SimpleCloser { _, watch, _ := n.watchable.Watch() namespaces := watch.Get() diff --git a/src/query/storage/m3/types.go b/src/query/storage/m3/types.go index 9413e92f19..99ef4f4410 100644 --- a/src/query/storage/m3/types.go +++ b/src/query/storage/m3/types.go @@ -27,8 +27,8 @@ import ( "github.com/m3db/m3/src/dbnode/namespace" genericstorage "github.com/m3db/m3/src/query/storage" "github.com/m3db/m3/src/query/storage/m3/consolidators" - xclose "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/instrument" + xresource "github.com/m3db/m3/src/x/resource" ) // Cleanup is a cleanup function to be called after resources are freed. @@ -116,7 +116,7 @@ type ClusterNamespacesWatcher interface { // RegisterListener registers a listener for updates to cluster namespaces. // If a value is currently present, it will synchronously call back the listener. - RegisterListener(listener ClusterNamespacesListener) xclose.SimpleCloser + RegisterListener(listener ClusterNamespacesListener) xresource.SimpleCloser // Close closes the watcher and all descendent watches. Close() diff --git a/src/x/checked/checked_mock.go b/src/x/checked/checked_mock.go index 2c5040ea5e..a546388273 100644 --- a/src/x/checked/checked_mock.go +++ b/src/x/checked/checked_mock.go @@ -144,10 +144,10 @@ func (mr *MockBytesMockRecorder) DecWrites() *gomock.Call { } // DelayFinalizer mocks base method -func (m *MockBytes) DelayFinalizer() resource.Closer { +func (m *MockBytes) DelayFinalizer() resource.SimpleCloser { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DelayFinalizer") - ret0, _ := ret[0].(resource.Closer) + ret0, _ := ret[0].(resource.SimpleCloser) return ret0 } diff --git a/src/x/checked/ref.go b/src/x/checked/ref.go index 569adbe9de..5257683a62 100644 --- a/src/x/checked/ref.go +++ b/src/x/checked/ref.go @@ -28,7 +28,7 @@ import ( "sync/atomic" "unsafe" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" ) // RefCount is an embeddable checked.Ref. @@ -103,14 +103,14 @@ func (c *RefCount) finalizeWithLock() { // until the closer returned by the method is called at least once. // This is useful for dependent resources requiring the lifetime of this // entityt to be extended. -func (c *RefCount) DelayFinalizer() resource.Closer { +func (c *RefCount) DelayFinalizer() xresource.SimpleCloser { c.finalizeState.Lock() c.finalizeState.delayRef++ c.finalizeState.Unlock() return c } -// Close implements resource.Closer for the purpose of use with DelayFinalizer. +// Close implements xresource.SimpleCloser for the purpose of use with DelayFinalizer. func (c *RefCount) Close() { c.finalizeState.Lock() c.finalizeState.delayRef-- diff --git a/src/x/checked/ref_test.go b/src/x/checked/ref_test.go index fe0a93827d..c17d751e95 100644 --- a/src/x/checked/ref_test.go +++ b/src/x/checked/ref_test.go @@ -30,13 +30,13 @@ import ( "testing" "time" - "github.com/m3db/m3/src/x/resource" - "github.com/leanovate/gopter" "github.com/leanovate/gopter/gen" "github.com/leanovate/gopter/prop" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + xresource "github.com/m3db/m3/src/x/resource" ) func TestRefCountNegativeRefCount(t *testing.T) { @@ -249,7 +249,7 @@ func TestRefCountDelayFinalizer(t *testing.T) { elem.IncRef() elem.DecRef() - delays := make([]resource.Closer, 0, test.numDelay) + delays := make([]xresource.SimpleCloser, 0, test.numDelay) for i := 0; i < test.numDelay; i++ { delays = append(delays, elem.DelayFinalizer()) } diff --git a/src/x/checked/types.go b/src/x/checked/types.go index 0b0e2f26b7..269fdb91d5 100644 --- a/src/x/checked/types.go +++ b/src/x/checked/types.go @@ -22,7 +22,7 @@ package checked import ( - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" ) // Ref is an entity that checks ref counts. @@ -43,7 +43,7 @@ type Ref interface { // until the closer returned by the method is called at least once. // This is useful for dependent resources requiring the lifetime of this // entityt to be extended. - DelayFinalizer() resource.Closer + DelayFinalizer() xresource.SimpleCloser // Finalize will call the finalizer if any, ref count must be zero. Finalize() diff --git a/src/x/context/context.go b/src/x/context/context.go index 5d72e11670..c6617e79e8 100644 --- a/src/x/context/context.go +++ b/src/x/context/context.go @@ -25,7 +25,7 @@ import ( "sync" xopentracing "github.com/m3db/m3/src/x/opentracing" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" lightstep "github.com/lightstep/lightstep-tracer-go" "github.com/opentracing/opentracing-go" @@ -52,8 +52,8 @@ type ctx struct { } type finalizeable struct { - finalizer resource.Finalizer - closer resource.Closer + finalizer xresource.Finalizer + closer xresource.SimpleCloser } // NewContext creates a new context. @@ -96,7 +96,7 @@ func (c *ctx) IsClosed() bool { return done } -func (c *ctx) RegisterFinalizer(f resource.Finalizer) { +func (c *ctx) RegisterFinalizer(f xresource.Finalizer) { parent := c.parentCtx() if parent != nil { parent.RegisterFinalizer(f) @@ -106,7 +106,7 @@ func (c *ctx) RegisterFinalizer(f resource.Finalizer) { c.registerFinalizeable(finalizeable{finalizer: f}) } -func (c *ctx) RegisterCloser(f resource.Closer) { +func (c *ctx) RegisterCloser(f xresource.SimpleCloser) { parent := c.parentCtx() if parent != nil { parent.RegisterCloser(f) diff --git a/src/x/context/context_test.go b/src/x/context/context_test.go index f5a36c8c44..b2afe6abfe 100644 --- a/src/x/context/context_test.go +++ b/src/x/context/context_test.go @@ -27,12 +27,12 @@ import ( "testing" "time" - "github.com/m3db/m3/src/x/resource" - "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/mocktracer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + xresource "github.com/m3db/m3/src/x/resource" ) func TestRegisterFinalizerWithChild(t *testing.T) { @@ -48,7 +48,7 @@ func TestRegisterFinalizerWithChild(t *testing.T) { ) wg.Add(1) - childCtx.RegisterFinalizer(resource.FinalizerFn(func() { + childCtx.RegisterFinalizer(xresource.FinalizerFn(func() { childClosed = true wg.Done() })) @@ -71,7 +71,7 @@ func TestRegisterFinalizer(t *testing.T) { ) wg.Add(1) - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { closed = true wg.Done() })) @@ -97,7 +97,7 @@ func TestRegisterCloserWithChild(t *testing.T) { ) wg.Add(1) - childCtx.RegisterCloser(resource.CloserFn(func() { + childCtx.RegisterCloser(xresource.SimpleCloserFn(func() { childClosed = true wg.Done() })) @@ -120,7 +120,7 @@ func TestRegisterCloser(t *testing.T) { ) wg.Add(1) - ctx.RegisterCloser(resource.CloserFn(func() { + ctx.RegisterCloser(xresource.SimpleCloserFn(func() { closed = true wg.Done() })) @@ -136,7 +136,7 @@ func TestRegisterCloser(t *testing.T) { func TestDoesNotRegisterFinalizerWhenClosed(t *testing.T) { ctx := NewContext().(*ctx) ctx.Close() - ctx.RegisterFinalizer(resource.FinalizerFn(func() {})) + ctx.RegisterFinalizer(xresource.FinalizerFn(func() {})) assert.Equal(t, 0, ctx.numFinalizeables()) } @@ -145,7 +145,7 @@ func TestDoesNotCloseTwice(t *testing.T) { ctx := NewContext().(*ctx) var closed int32 - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { atomic.AddInt32(&closed, 1) })) @@ -187,7 +187,7 @@ func testDependsOn(t *testing.T, c *ctx) { other := NewContext().(*ctx) wg.Add(1) - c.RegisterFinalizer(resource.FinalizerFn(func() { + c.RegisterFinalizer(xresource.FinalizerFn(func() { atomic.AddInt32(&closed, 1) wg.Done() })) @@ -221,7 +221,7 @@ func TestDependsOnWithChild(t *testing.T) { ) wg.Add(1) - c.RegisterFinalizer(resource.FinalizerFn(func() { + c.RegisterFinalizer(xresource.FinalizerFn(func() { atomic.AddInt32(&closed, 1) wg.Done() })) diff --git a/src/x/context/pool_test.go b/src/x/context/pool_test.go index 7757171243..a2106a4995 100644 --- a/src/x/context/pool_test.go +++ b/src/x/context/pool_test.go @@ -23,7 +23,7 @@ package context import ( "testing" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" "github.com/stretchr/testify/assert" ) @@ -34,7 +34,7 @@ func TestContextPool(t *testing.T) { ctx := pool.Get() finalizeCalled := false - ctx.RegisterFinalizer(resource.FinalizerFn(func() { + ctx.RegisterFinalizer(xresource.FinalizerFn(func() { finalizeCalled = true })) ctx.BlockingClose() diff --git a/src/x/context/types.go b/src/x/context/types.go index bf152f05ca..d384dc3411 100644 --- a/src/x/context/types.go +++ b/src/x/context/types.go @@ -24,7 +24,7 @@ import ( stdctx "context" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/resource" + xresource "github.com/m3db/m3/src/x/resource" "github.com/opentracing/opentracing-go" ) @@ -47,10 +47,10 @@ type Context interface { IsClosed() bool // RegisterFinalizer will register a resource finalizer. - RegisterFinalizer(resource.Finalizer) + RegisterFinalizer(xresource.Finalizer) // RegisterCloser will register a resource closer. - RegisterCloser(resource.Closer) + RegisterCloser(xresource.SimpleCloser) // DependsOn will register a blocking context that // must complete first before finalizers can be called. diff --git a/src/x/close/close.go b/src/x/resource/close.go similarity index 70% rename from src/x/close/close.go rename to src/x/resource/close.go index 1b652005fa..668674127f 100644 --- a/src/x/close/close.go +++ b/src/x/resource/close.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016 Uber Technologies, Inc. +// Copyright (c) 2020 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -18,12 +18,12 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -// Package close provides utilities for closing resources. -package close +package resource import ( "errors" - "io" + + xerrors "github.com/m3db/m3/src/x/errors" ) var ( @@ -32,32 +32,6 @@ var ( ErrNotCloseable = errors.New("not a closeable resource") ) -// Closer is a resource that can be closed. -type Closer interface { - io.Closer -} - -// CloserFn implements the SimpleCloser interface. -type CloserFn func() error - -// Close implements the SimplerCloser interface. -func (fn CloserFn) Close() error { - return fn() -} - -// SimpleCloser is a resource that can be closed without returning a result. -type SimpleCloser interface { - Close() -} - -// SimpleCloserFn implements the SimpleCloser interface. -type SimpleCloserFn func() - -// Close implements the SimplerCloser interface. -func (fn SimpleCloserFn) Close() { - fn() -} - // TryClose attempts to close a resource, the resource is expected to // implement either Closeable or CloseableResult. func TryClose(r interface{}) error { @@ -70,3 +44,15 @@ func TryClose(r interface{}) error { } return ErrNotCloseable } + +// CloseAll closes all closers and combines any errors. +func CloseAll(closers ...Closer) error { + multiErr := xerrors.NewMultiError() + for _, closer := range closers { + if err := closer.Close(); err != nil { + multiErr = multiErr.Add(err) + } + } + + return multiErr.FinalError() +} diff --git a/src/x/close/close_test.go b/src/x/resource/close_test.go similarity index 92% rename from src/x/close/close_test.go rename to src/x/resource/close_test.go index 926df11e76..a8ab96103b 100644 --- a/src/x/close/close_test.go +++ b/src/x/resource/close_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Uber Technologies, Inc. +// Copyright (c) 2020 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -17,14 +17,13 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. - -package close +package resource import ( "errors" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTryClose(t *testing.T) { @@ -53,10 +52,10 @@ func TestTryClose(t *testing.T) { for _, test := range tests { err := TryClose(test.input) if test.expectErr { - assert.Error(t, err) + require.Error(t, err) continue } - assert.NoError(t, err) + require.NoError(t, err) } } diff --git a/src/x/resource/types.go b/src/x/resource/types.go index 05933c593d..42b2b0d02c 100644 --- a/src/x/resource/types.go +++ b/src/x/resource/types.go @@ -38,15 +38,28 @@ func (fn FinalizerFn) Finalize() { fn() } -// Closer is an object that can be closed. -type Closer interface { +// SimpleCloser is an object that can be closed. +type SimpleCloser interface { Close() } -// CloserFn is a function literal that is a closer. -type CloserFn func() +// SimpleCloserFn is a function literal that is a closer. +type SimpleCloserFn func() // Close will call the function literal as a closer. -func (fn CloserFn) Close() { +func (fn SimpleCloserFn) Close() { fn() } + +// Closer is an object that can be closed which returns an error. +type Closer interface { + Close() error +} + +// CloserFn is a function literal that is a closer which returns an error. +type CloserFn func() error + +// Close will call the function literal as a closer. +func (fn CloserFn) Close() error { + return fn() +} diff --git a/src/x/watch/source.go b/src/x/watch/source.go index 4d174e1cb2..aba93b0087 100644 --- a/src/x/watch/source.go +++ b/src/x/watch/source.go @@ -24,7 +24,7 @@ import ( "errors" "sync" - "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" "go.uber.org/zap" ) @@ -41,7 +41,7 @@ type SourceInput interface { // Source polls data by calling SourcePollFn and notifies its watches on updates. type Source interface { - close.SimpleCloser + xresource.SimpleCloser // Get returns the latest value. Get() interface{} diff --git a/src/x/watch/watch.go b/src/x/watch/watch.go index 42e531fc77..5b35b17578 100644 --- a/src/x/watch/watch.go +++ b/src/x/watch/watch.go @@ -25,7 +25,7 @@ import ( "errors" "sync" - xclose "github.com/m3db/m3/src/x/close" + xresource "github.com/m3db/m3/src/x/resource" ) var errClosed = errors.New("closed") @@ -34,7 +34,7 @@ type closer func() // Updatable can be updated. type Updatable interface { - xclose.SimpleCloser + xresource.SimpleCloser // C returns the notification channel for updates. C() <-chan struct{} @@ -50,7 +50,7 @@ type Watch interface { // Watchable can be watched type Watchable interface { - xclose.SimpleCloser + xresource.SimpleCloser // IsClosed returns true if the Watchable is closed IsClosed() bool