From 31bc9727538b9f69b4f786e92173ab114d1171c2 Mon Sep 17 00:00:00 2001 From: Johann Wagner Date: Thu, 2 Dec 2021 23:21:17 +0100 Subject: [PATCH] Introducing shard_index to overcome limitations of sorting --- databases/redis.go | 33 ++++++++++++++++++-------- settings/README.md | 59 +++++++++++++++++++++++++--------------------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/databases/redis.go b/databases/redis.go index 8b365b9..273db6d 100644 --- a/databases/redis.go +++ b/databases/redis.go @@ -34,7 +34,7 @@ import ( type Redis struct { metricsPrefix string redisDurations *prometheus.HistogramVec - clients []redis.UniversalClient + clients map[uint32]redis.UniversalClient pipeline redis.Pipeliner mutex sync.Mutex channel chan bool @@ -91,6 +91,7 @@ type RedisSettings struct { Password string `json:"password"` SentinelUsername string `json:"sentinel_username"` SentinelPassword string `json:"sentinel_password"` + ShardIndex int64 `json:"shard_index"` } type MetricHook struct { @@ -173,6 +174,13 @@ var RedisForm = forms.Form{ forms.IsString{}, }, }, + { + Name: "shard_index", + Validators: []forms.Validator{ + forms.IsOptional{Default: 0}, + forms.IsInteger{}, + }, + }, }, } var RedisShardForm = forms.Form{ @@ -215,7 +223,7 @@ func ValidateRedisShardSettings(settings map[string]interface{}) (interface{}, e } } -func MakeRedisClient(settings interface{}) (redis.UniversalClient, error) { +func MakeRedisClient(settings interface{}) (redis.UniversalClient, uint32, error) { redisSettings := settings.(RedisSettings) var client redis.UniversalClient @@ -250,19 +258,20 @@ func MakeRedisClient(settings interface{}) (redis.UniversalClient, error) { services.Log.Info("Creating sentinel-based redis connection") } else { - return nil, errors.New("invalid database configuration, needed addresses or sentinelAddresses") + return nil, 0, errors.New("invalid database configuration, needed addresses or sentinelAddresses") } - return client, nil + return client, uint32(redisSettings.ShardIndex), nil } func MakeRedisShards(settings interface{}) (*Redis, error) { redisShardSettings := settings.(RedisShardSettings) ctx := context.TODO() - clients := []redis.UniversalClient{} + clients := map[uint32]redis.UniversalClient{} + for _, redisSettings := range redisShardSettings.Shards { - client, err := MakeRedisClient(redisSettings) + client, shardIndex, err := MakeRedisClient(redisSettings) if err != nil { return nil, err @@ -272,7 +281,7 @@ func MakeRedisShards(settings interface{}) (*Redis, error) { return nil, err } - clients = append(clients, client) + clients[shardIndex] = client } services.Log.Info("Creating redis-shard database") @@ -290,7 +299,7 @@ func MakeRedisShards(settings interface{}) (*Redis, error) { func MakeRedis(settings interface{}) (*Redis, error) { ctx := context.TODO() - client, err := MakeRedisClient(settings) + client, shardIndex, err := MakeRedisClient(settings) if err != nil { return nil, err @@ -302,8 +311,11 @@ func MakeRedis(settings interface{}) (*Redis, error) { services.Log.Info("Creating redis database") + var clients map[uint32]redis.UniversalClient + clients[shardIndex] = client + database := &Redis{ - clients: []redis.UniversalClient{client}, + clients: clients, channel: make(chan bool), Ctx: ctx, } @@ -362,7 +374,8 @@ func (d *Redis) getShardForKey(key string) uint32 { } func (d *Redis) Client(key string) redis.UniversalClient { - return d.clients[d.getShardForKey(key)] + shard_index := d.getShardForKey(key) + return d.clients[shard_index] } func (d *Redis) Open() error { diff --git a/settings/README.md b/settings/README.md index 8658569..ee6b888 100644 --- a/settings/README.md +++ b/settings/README.md @@ -1,60 +1,65 @@ # Configuration Options -The application gets configured by a set of configuration files. `001_default.yaml` contains the main configuration, administration commands will create the other configurations based on the `001_default.yaml` configuration. +The application gets configured by a set of configuration files. `001_default.yaml` contains the main configuration, +administration commands will create the other configurations based on the `001_default.yaml` configuration. ## Databases It is recommended to use a redis database. You need two different databases, i.e. for the application and the metrics. -The `metrics` key contains the database configuration for the metrics and the `database` key contains the database configuration for the application. +The `metrics` key contains the database configuration for the metrics and the `database` key contains the database +configuration for the application. ### Redis -You can configure the application to use a standalone redis instance. +You can configure the application to use a standalone redis instance. ```yaml name: db # or meter type: redis settings: - addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server. - database: 1 # Redis database ID - password: "" # Redis password - master: "mymaster" # Redis master name + addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server. + database: 1 # Redis database ID + password: "" # Redis password + master: "mymaster" # Redis master name ``` ### Redis + Sentinel -You can configure the application to use a redis + sentinel instance. +You can configure the application to use a redis + sentinel instance. ```yaml name: db # or meter type: redis settings: - sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server. - database: 1 # Redis database ID - password: "" # Redis password - master: "mymaster" # Redis master name - sentinel_username: "username" # Sentinel username - sentinel_password: "password" # Sentinel password + sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server. + database: 1 # Redis database ID + password: "" # Redis password + master: "mymaster" # Redis master name + sentinel_username: "username" # Sentinel username + sentinel_password: "password" # Sentinel password ``` - ### Redis Sharded -You can also use the application-based Redis sharding by specifing multiple shards. You can use Redis standalone or Redis + Sentinel connections. The application will automagically shard the Redis keys to the different Redis instances. Make sure, that the configuration order is consistent, otherwise the database gets mixed up. +You can also use the application-based Redis sharding by specifing multiple shards. You can use Redis standalone or +Redis + Sentinel connections. The application will automagically shard the Redis keys to the different Redis instances. +Make sure, that the shard_index is consistent, otherwise the database gets mixed up. ```yaml name: db # or meter type: redis-shard settings: - shards: - - addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server. - database: 1 # Redis database ID - password: "" # Redis password - master: "mymaster" # Redis master name - - sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server. - database: 1 # Redis database ID - password: "" # Redis password - master: "mymaster" # Redis master name - sentinel_username: "username" # Sentinel username - sentinel_password: "password" # Sentinel password + shards: + - addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server. + database: 1 # Redis database ID + password: "" # Redis password + master: "mymaster" # Redis master name + shard_index: 0 # Ascending shard index, beginning at 0 + - sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server. + database: 1 # Redis database ID + password: "" # Redis password + master: "mymaster" # Redis master name + sentinel_username: "username" # Sentinel username + sentinel_password: "password" # Sentinel password + shard_index: 1 # Ascending shard index, beginning at 0 ``` \ No newline at end of file