Introducing shard_index to overcome limitations of sorting

OpenHCS · Dec 2, 2021 · 31bc972 · 31bc972
1 parent 18b59a9
commit 31bc972
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 37 deletions.
diff --git a/databases/redis.go b/databases/redis.go
@@ -34,7 +34,7 @@ import (
 type Redis struct {
 	metricsPrefix  string
 	redisDurations *prometheus.HistogramVec
-	clients        []redis.UniversalClient
+	clients        map[uint32]redis.UniversalClient
 	pipeline       redis.Pipeliner
 	mutex          sync.Mutex
 	channel        chan bool
@@ -91,6 +91,7 @@ type RedisSettings struct {
 	Password          string   `json:"password"`
 	SentinelUsername  string   `json:"sentinel_username"`
 	SentinelPassword  string   `json:"sentinel_password"`
+	ShardIndex        int64    `json:"shard_index"`
 }
 
 type MetricHook struct {
@@ -173,6 +174,13 @@ var RedisForm = forms.Form{
 				forms.IsString{},
 			},
 		},
+		{
+			Name: "shard_index",
+			Validators: []forms.Validator{
+				forms.IsOptional{Default: 0},
+				forms.IsInteger{},
+			},
+		},
 	},
 }
 var RedisShardForm = forms.Form{
@@ -215,7 +223,7 @@ func ValidateRedisShardSettings(settings map[string]interface{}) (interface{}, e
 	}
 }
 
-func MakeRedisClient(settings interface{}) (redis.UniversalClient, error) {
+func MakeRedisClient(settings interface{}) (redis.UniversalClient, uint32, error) {
 	redisSettings := settings.(RedisSettings)
 
 	var client redis.UniversalClient
@@ -250,19 +258,20 @@ func MakeRedisClient(settings interface{}) (redis.UniversalClient, error) {
 
 		services.Log.Info("Creating sentinel-based redis connection")
 	} else {
-		return nil, errors.New("invalid database configuration, needed addresses or sentinelAddresses")
+		return nil, 0, errors.New("invalid database configuration, needed addresses or sentinelAddresses")
 	}
 
-	return client, nil
+	return client, uint32(redisSettings.ShardIndex), nil
 }
 
 func MakeRedisShards(settings interface{}) (*Redis, error) {
 	redisShardSettings := settings.(RedisShardSettings)
 	ctx := context.TODO()
 
-	clients := []redis.UniversalClient{}
+	clients := map[uint32]redis.UniversalClient{}
+
 	for _, redisSettings := range redisShardSettings.Shards {
-		client, err := MakeRedisClient(redisSettings)
+		client, shardIndex, err := MakeRedisClient(redisSettings)
 
 		if err != nil {
 			return nil, err
@@ -272,7 +281,7 @@ func MakeRedisShards(settings interface{}) (*Redis, error) {
 			return nil, err
 		}
 
-		clients = append(clients, client)
+		clients[shardIndex] = client
 	}
 
 	services.Log.Info("Creating redis-shard database")
@@ -290,7 +299,7 @@ func MakeRedisShards(settings interface{}) (*Redis, error) {
 func MakeRedis(settings interface{}) (*Redis, error) {
 	ctx := context.TODO()
 
-	client, err := MakeRedisClient(settings)
+	client, shardIndex, err := MakeRedisClient(settings)
 
 	if err != nil {
 		return nil, err
@@ -302,8 +311,11 @@ func MakeRedis(settings interface{}) (*Redis, error) {
 
 	services.Log.Info("Creating redis database")
 
+	var clients map[uint32]redis.UniversalClient
+	clients[shardIndex] = client
+
 	database := &Redis{
-		clients: []redis.UniversalClient{client},
+		clients: clients,
 		channel: make(chan bool),
 		Ctx:     ctx,
 	}
@@ -362,7 +374,8 @@ func (d *Redis) getShardForKey(key string) uint32 {
 }
 
 func (d *Redis) Client(key string) redis.UniversalClient {
-	return d.clients[d.getShardForKey(key)]
+	shard_index := d.getShardForKey(key)
+	return d.clients[shard_index]
 }
 
 func (d *Redis) Open() error {

diff --git a/settings/README.md b/settings/README.md
@@ -1,60 +1,65 @@
 # Configuration Options
 
-The application gets configured by a set of configuration files. `001_default.yaml` contains the main configuration, administration commands will create the other configurations based on the `001_default.yaml` configuration.
+The application gets configured by a set of configuration files. `001_default.yaml` contains the main configuration,
+administration commands will create the other configurations based on the `001_default.yaml` configuration.
 
 ## Databases
 
 It is recommended to use a redis database. You need two different databases, i.e. for the application and the metrics.
-The `metrics` key contains the database configuration for the metrics and the `database` key contains the database configuration for the application.
+The `metrics` key contains the database configuration for the metrics and the `database` key contains the database
+configuration for the application.
 
 ### Redis
 
-You can configure the application to use a standalone redis instance. 
+You can configure the application to use a standalone redis instance.
 
 ```yaml
 name: db # or meter
 type: redis
 settings:
-    addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server.
-    database: 1 # Redis database ID
-    password: "" # Redis password
-    master: "mymaster" # Redis master name
+  addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server.
+  database: 1 # Redis database ID
+  password: "" # Redis password
+  master: "mymaster" # Redis master name
 ```
 
 ### Redis + Sentinel
 
-You can configure the application to use a redis + sentinel instance. 
+You can configure the application to use a redis + sentinel instance.
 
 ```yaml
 name: db # or meter
 type: redis
 settings:
-    sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server.
-    database: 1 # Redis database ID
-    password: "" # Redis password
-    master: "mymaster" # Redis master name
-    sentinel_username: "username" # Sentinel username
-    sentinel_password: "password" # Sentinel password
+  sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server.
+  database: 1 # Redis database ID
+  password: "" # Redis password
+  master: "mymaster" # Redis master name
+  sentinel_username: "username" # Sentinel username
+  sentinel_password: "password" # Sentinel password
 ```
 
-
 ### Redis Sharded
 
-You can also use the application-based Redis sharding by specifing multiple shards. You can use Redis standalone or Redis + Sentinel connections. The application will automagically shard the Redis keys to the different Redis instances. Make sure, that the configuration order is consistent, otherwise the database gets mixed up.
+You can also use the application-based Redis sharding by specifing multiple shards. You can use Redis standalone or
+Redis + Sentinel connections. The application will automagically shard the Redis keys to the different Redis instances.
+Make sure, that the shard_index is consistent, otherwise the database gets mixed up.
 
 ```yaml
 name: db # or meter
 type: redis-shard
 settings:
-    shards:
-      - addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server.
-        database: 1 # Redis database ID
-        password: "" # Redis password
-        master: "mymaster" # Redis master name 
-      - sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server.
-        database: 1 # Redis database ID
-        password: "" # Redis password
-        master: "mymaster" # Redis master name
-        sentinel_username: "username" # Sentinel username
-        sentinel_password: "password" # Sentinel password
+  shards:
+    - addresses: [ "localhost:6379" ] # Set of addresses pointing to the SAME redis server.
+      database: 1 # Redis database ID
+      password: "" # Redis password
+      master: "mymaster" # Redis master name 
+      shard_index: 0  # Ascending shard index, beginning at 0
+    - sentinel_addresses: [ "localhost:26379" ] # Set of addresses pointing to the SAME sentinel server.
+      database: 1 # Redis database ID
+      password: "" # Redis password
+      master: "mymaster" # Redis master name
+      sentinel_username: "username" # Sentinel username
+      sentinel_password: "password" # Sentinel password
+      shard_index: 1 # Ascending shard index, beginning at 0
 ```