Merge branch 'develop'

README.md

@@ -103,15 +103,17 @@ sh repoll-init.sh repoll # 密码自定义
 * 管理员（dba角色）进行配置上线
 
 # demo演示
-http://repoll.club:8091/
+http://43.143.240.39/
 admin/admin
 
 
 # Todo list
 - [x] 支持哨兵模式和集群模式
 - [x] 监控独立展示，包括qps、内存使用率、客户端链接以及慢查询等
 - [x] 支持在线扩缩容的申请、审批、配置生效流程
+- [x] 支持Redis密码
 - [x] 支持导入已存在的redis实例
+- [ ] 支持不同Redis集群间的数据迁移、同步、校验等运管功能
 - [ ] 支持web console，在线执行redis命令
 - [ ] 支持容器化部署
 - [ ] 支持Redis实例容器化部署

polls/admin.py

@@ -406,19 +406,19 @@ def get_actions(self, request):
                 del actions['delete_selected']
         return actions
 
-    # def get_queryset(self, request):
-    #     """函数作用：使当前登录的用户只能看到自己负责的实例"""
-    #     qs = super(RedisApplyAdmin, self).get_queryset(request)
-    #     if request.user.is_superuser:
-    #         return qs
-    #     return qs.filter(create_user=RedisApply.objects.filter(create_user=request.user))
-
-    # def has_change_permission(self, request, obj=None):
-    #     if obj:
-    #         if request.method not in ('GET', 'HEAD'):
-    #             self.message_user(request, "操作实例为 {0} 的实例失败，实例已存在无法修改".format(obj.apply_ins_name))
-    #             return False
-    #     return super(RedisApplyAdmin, self).has_change_permission(request, obj)
+    def get_queryset(self, request):
+        """函数作用：使当前登录的用户只能看到自己负责的实例"""
+        qs = super(RedisApplyAdmin, self).get_queryset(request)
+        if request.user.is_superuser:
+            return qs
+        return qs.filter(create_user=RedisApply.objects.filter(create_user=request.user))
+
+    def has_change_permission(self, request, obj=None):
+        if obj:
+            if request.method not in ('GET', 'HEAD'):
+                self.message_user(request, "操作实例为 {0} 的实例失败，实例已存在无法修改".format(obj.apply_ins_name))
+                return False
+        return super(RedisApplyAdmin, self).has_change_permission(request, obj)
 
     list_display = ['id', 'apply_ins_name', 'ins_disc', 'redis_type',
                     'redis_mem', 'sys_author', 'area',
@@ -739,6 +739,7 @@ def has_delete_permission(self, request, obj=None):
     list_display = ['id', 'ip', 'area', 'machina_type', 'machina_mem']
     list_filter = ['ip', 'area']
     search_fields = ['ip', 'area']
+    list_display_links = ['id', 'ip']
     list_per_page = 15
 
     inlines = [ServerUserLine]

polls/apis.py

@@ -16,24 +16,33 @@ class Meta:
         fields = '__all__'
 
 
+def get_running_ins(redis_type):
+    if redis_type == 'sentinel':
+        running_ins_time = RunningInsSentinel.objects.all()
+    elif redis_type == 'standalone':
+        running_ins_time = RunningInsStandalone.objects.all()
+    elif redis_type == 'cluster':
+        running_ins_time = RunningInsCluster.objects.all()
+    else:
+        running_ins_time = "default"
+    return running_ins_time
+
+
 @api_view(['GET'])
 @permission_classes((permissions.IsAuthenticated,))
 def redisstop(request, redis_type, ins_id):
     """
+    TODO：硬编码密码
     API接口，停止redis实例。
     授权模式，当前平台内部用户
     """
-    if redis_type == 'sentinel':
-        running_ins_time = RunningInsSentinel.objects.all()
-    elif redis_type == 'standalone':
-        running_ins_time = RunningInsStandalone.objects.all()
-    elif redis_type == 'cluster':
-        running_ins_time = RunningInsCluster.objects.all()
+    running_ins_time = get_running_ins(redis_type)
     running_ins = running_ins_time.filter(id=ins_id)
     running_ins_ip = running_ins.values('redis_ip').first()
     running_ins_port = running_ins.values('running_ins_port').first()
     redisins = RedisStartClass(host=running_ins_ip['redis_ip'],
-                               redis_server_ctl="/opt/repoll/redis/src/redis-cli -p {0} shutdown".format(running_ins_port['running_ins_port']))
+                               redis_server_ctl="/opt/repoll/redis/src/redis-cli -a qZr3pet -p {0} shutdown".format(
+                                   running_ins_port['running_ins_port']))
     serializer = RunningInsTimeSerializer(running_ins, many=True)
     result = serializer.data[0]
     if redisins.start_server():
@@ -62,25 +71,22 @@ def redisstart(request, redis_type, ins_id):
     API接口，启动redis实例。
     授权模式，当前平台内部用户
     """
-    if redis_type == 'sentinel':
-        running_ins_time = RunningInsSentinel.objects.all()
-    elif redis_type == 'standalone':
-        running_ins_time = RunningInsStandalone.objects.all()
-    elif redis_type == 'cluster':
-        running_ins_time = RunningInsCluster.objects.all()
+    running_ins_time = get_running_ins(redis_type)
     running_ins = running_ins_time.filter(id=ins_id)
     running_ins_ip = running_ins.values('redis_ip').first()
     running_ins_port = running_ins.values('running_ins_port').first()
     for c in running_ins:
         running_ins_type = c.__dict__['redis_type']
     if running_ins_type == 'Redis-Sentinel':
         redisins = RedisStartClass(host=running_ins_ip['redis_ip'],
-                                   redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/{0}-sentienl.conf --sentinel".format(
+                                   redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/"
+                                                    "{0}-sentinel.conf --sentinel".format(
                                        running_ins_port['running_ins_port']))
     else:
         if redis_type == 'cluster':
             redisins = RedisStartClass(host=running_ins_ip['redis_ip'],
-                                       redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/{0}-cluster.conf".format(
+                                       redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/"
+                                                        "{0}-cluster.conf".format(
                                            running_ins_port['running_ins_port']))
         else:
             redisins = RedisStartClass(host=running_ins_ip['redis_ip'],
@@ -121,7 +127,7 @@ def allredisins(request, redis_type=None):
     try:
         if redis_type == "all":
             sentinel_ins = sentinel_ins_time.values('redis_ip', 'running_ins_port', 'redis_ins_alive',
-                                                    'redis_ins_mem', 'redis_type', 'running_ins_name',)
+                                                    'redis_ins_mem', 'redis_type', 'running_ins_name', )
             stanalone_ins = stanalone_ins_time.values('redis_ip', 'running_ins_port', 'redis_ins_alive',
                                                       'redis_ins_mem', 'redis_type', 'running_ins_name')
             cluster_ins = cluster_ins_time.values('redis_ip', 'running_ins_port', 'redis_ins_alive',
@@ -324,4 +330,4 @@ def import_ext_ins(request):
     except IOError as e:
         result['redis_status'] = False
         return Response(result)
-    return Response(result)
+    return Response(result)

polls/handlers.py

@@ -1,7 +1,7 @@
 from .models import *
 import paramiko
 import logging
-from .scheduled import mem_unit_chage
+from .scheduled import mem_unit_change
 from django.core.exceptions import ValidationError
 # 针对model 的signal
 from django.dispatch import receiver
@@ -35,20 +35,20 @@ def apply_redis_text_handler(sender, **kwargs):
     if redis_ins_type == 'Redis-Standalone':
         redis_ip = redis_apply_text_split['redis_ip']
         redis_port = redis_apply_text_split['redis_port']
-        redis_standalon_ins = RedisStandalone(redis_ins=redis_ins_obj,
-                                              redis_ins_name=redis_ins_obj_name,
-                                              redis_ins_type=redis_ins_type,
-                                              redis_ins_mem=redis_apply_text_split['redis_mem'],
-                                              redis_ip=redis_ip,
-                                              redis_port=redis_port)
-        redis_standalon_ins.saved_redis_running_ins()
-        if redis_standalon_ins.create_redis_conf_file():
+        redis_standalone_ins = RedisStandalone(redis_ins=redis_ins_obj,
+                                               redis_ins_name=redis_ins_obj_name,
+                                               redis_ins_type=redis_ins_type,
+                                               redis_ins_mem=redis_apply_text_split['redis_mem'],
+                                               redis_ip=redis_ip,
+                                               redis_port=redis_port)
+        redis_standalone_ins.saved_redis_running_ins()
+        if redis_standalone_ins.create_redis_conf_file():
             redis_start = RedisStartClass(host=redis_ip,
                                           redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/" + str(redis_port) + ".conf")
             if redis_start.start_server():
                 logger.info("Redis 单实例启动成功，服务器IP：{0}, 启动端口为：{1}".format(redis_ip, redis_port))
             else:
-                logging.info("Redis 单实例启动失败，服务器IP：{0}, 启动端口为：{1}".format(redis_ip, redis_port))
+                logger.info("Redis 单实例启动失败，服务器IP：{0}, 启动端口为：{1}".format(redis_ip, redis_port))
                 raise ValidationError("redis 单实例启动失败")
         else:
             raise ValidationError("redis 单实例启动失败")
@@ -72,13 +72,13 @@ def apply_redis_text_handler(sender, **kwargs):
                 start_slave = b.start_slave_master()
                 if start_slave:
                     b.start_sentinel_master()
-                    logging.info("哨兵模式启动成功,redis_master_ip_port:{0},"
-                                 "redis_slave_ip_port:{1},"
-                                 "redis_sentinel_ip_port:{2},"
-                                 "redis_master_name:{3}".format(redis_apply_text_split['redis_master_ip_port'],
-                                                                redis_apply_text_split['redis_slave_ip_port'],
-                                                                redis_apply_text_split['redis_sentinel_ip_port'],
-                                                                redis_apply_text_split['redis_master_name']))
+                    logger.info("哨兵模式启动成功,redis_master_ip_port:{0},"
+                                "redis_slave_ip_port:{1},"
+                                "redis_sentinel_ip_port:{2},"
+                                "redis_master_name:{3}".format(redis_apply_text_split['redis_master_ip_port'],
+                                                               redis_apply_text_split['redis_slave_ip_port'],
+                                                               redis_apply_text_split['redis_sentinel_ip_port'],
+                                                               redis_apply_text_split['redis_master_name']))
         else:
             raise ValidationError("redis 哨兵动失败")
         b.save_sentinel_redis_ins()
@@ -105,6 +105,7 @@ def apply_redis_text_handler(sender, **kwargs):
                                           redis_ins_mem=redis_one_ins['redis_mem'],
                                           redis_ip=all_redis_ins[0],
                                           redis_port=all_redis_ins[1])
+                    # c.check_redis_cluster_port(redis_apply_text_split)
                     file_status = c.create_cluster_file()
                     if file_status:
                         c.start_all_redis_ins()
@@ -355,8 +356,10 @@ def regx_redis_conf(key, value, port, maxmemory=None, **kwargs):
                 key = key.replace(key, "sentinel monitor ")
                 value = value.replace("%masterName_ip_port_num%",
                                       " {0} {1} {2} {3}".format(kwargs['kwargs']['masterName'], kwargs['kwargs']['masterIp'],
-                                                                kwargs['kwargs']['masterPort'], kwargs['kwargs']['sentienlNum']))
+                                                                kwargs['kwargs']['masterPort'], kwargs['kwargs']['sentinelNum']))
                 return key, value
+            elif "authPass" in key:
+                key = key.replace(key, "sentinel auth-pass {0} ".format(kwargs['kwargs']['masterName']))
             elif "sentinelDownAfterMilliseconds" in key:
                 key = key.replace(key, "sentinel down-after-milliseconds ")
                 value = value.replace(value, " {0} 20000".format(kwargs['kwargs']['masterName']))
@@ -421,8 +424,6 @@ def saved_redis_running_ins(self):
         obj_runningins = RunningInsTime(running_ins_name=self.redis_ins_name,
                                         redis_type=self.redis_ins_type,
                                         redis_ins_mem=self.redis_ins_mem,
-                                        # redis_ip=self.redis_ip,
-                                        # running_ins_port=self.redis_port
                                         )
         obj_runningins.save()
         try:
@@ -456,7 +457,7 @@ def create_redis_conf_file(self):
                 if k != 'id' and k != 'redis_version' and k != 'redis_type':
                     if isinstance(v, str) or isinstance(v, int):
                         k, v = regx_redis_conf(key=k, value=v, port=self.redis_port,
-                                               maxmemory=mem_unit_chage(self.redis_ins_mem))
+                                               maxmemory=mem_unit_change(self.redis_ins_mem))
                         f.write(k + " " + str(v) + "\n")
             if self.master_name:
                 _maser_ip_port = self.master_ip_port.split(":")
@@ -542,12 +543,12 @@ def create_sentienl_conf_file(self):
         for sentinel in self.redis_sentinel_ip_port:
             if isinstance(sentinel, str):
                 redis_sentinel_ip, redis_sentinel_port = sentinel.split(":")
-                conf_file_name = "{0}/templates/".format(TEMPLATES_DIR) + str(redis_sentinel_port) + "-sentienl.conf"
+                conf_file_name = "{0}/templates/".format(TEMPLATES_DIR) + str(redis_sentinel_port) + "-sentinel.conf"
                 conf_modify = {
                     "masterName": self.redis_master_name,
                     "masterIp": self.redis_master_ip,
                     "masterPort": self.redis_master_port,
-                    "sentienlNum": self.redis_sentinel_num,
+                    "sentinelNum": self.redis_sentinel_num,
                 }
                 with open(conf_file_name, 'w+') as f:
                     for k, v in all_redis_conf[0].items():
@@ -557,7 +558,7 @@ def create_sentienl_conf_file(self):
                                                        port=redis_sentinel_port, kwargs=conf_modify)
                                 f.write(k + " " + str(v) + "\n")
                 if do_scp(redis_sentinel_ip, conf_file_name,
-                          "/opt/repoll/conf/" + str(redis_sentinel_port) + "-sentienl.conf"):
+                          "/opt/repoll/conf/" + str(redis_sentinel_port) + "-sentinel.conf"):
                     logging.info("文件分发成功")
                 else:
                     logging.error("文件分发失败")
@@ -624,7 +625,7 @@ def start_sentinel_master(self):
                 redis_sentinel_ip, redis_sentinel_port = sentinel.split(":")
                 redis_sentinel_start = RedisStartClass(host=redis_sentinel_ip,
                                                        redis_server_ctl="/opt/repoll/redis/src/redis-server /opt/repoll/conf/" +
-                                                                        str(redis_sentinel_port) + "-sentienl.conf --sentinel")
+                                                                        str(redis_sentinel_port) + "-sentinel.conf --sentinel")
                 redis_sentinel_start_result = redis_sentinel_start.start_server()
                 start_result_dict["{0}:{1}".format(redis_sentinel_ip, redis_sentinel_port)] = redis_sentinel_start_result
         return start_result_dict
@@ -715,13 +716,13 @@ def create_cluster_file(self):
                 if k != 'id' and k != 'redis_version' and k != 'redis_type':
                     if isinstance(v, str) or isinstance(v, int):
                         k, v = regx_redis_conf(key=k, value=v, port=self.redis_port,
-                                               maxmemory=mem_unit_chage(self.redis_ins_mem))
+                                               maxmemory=mem_unit_change(self.redis_ins_mem))
                         f.write(k + " " + str(v) + "\n")
             for k, v in all_cluster_conf[0].items():
                 if k != 'id' and k != 'redis_version' and k != 'redis_type':
                     if isinstance(v, str) or isinstance(v, int):
                         k, v = regx_redis_conf(key=k, value=v, port=self.redis_port,
-                                               maxmemory=mem_unit_chage(self.redis_ins_mem),
+                                               maxmemory=mem_unit_change(self.redis_ins_mem),
                                                kwargs={"redis_port": self.redis_port})
                         f.write(k + " " + str(v) + "\n")
         if do_scp(self.redis_ip, conf_file_name, "/opt/repoll/conf/" + str(self.redis_port) + "-cluster.conf"):
@@ -746,6 +747,22 @@ def start_all_redis_ins(self):
             logger.info("redis 实例{2}启动失败，ip:port: {0}:{1}".format(self.redis_ip, self.redis_port, self.redis_ins_name))
             return False
 
+    def check_redis_cluster_port(self, redis_one_ins):
+        """
+        由于集群需要使用监听端口和集群间通信2个端口，默认逻辑是通信端口=监听端口+10000
+        :param redis_one_ins: 集群实例详情
+        :return:
+        """
+        for redis_ins in redis_one_ins:
+            for ins in redis_ins["redis_ip_port"]:
+                cluster_self_connet_port = int(ins[1]) + 10000
+                _comm = f"/usr/sbin/lsof -i:{cluster_self_connet_port}"
+                _ex_comm = do_command(ins[0], _comm)
+                if _ex_comm[0] == 0:
+                    logger.error(f"{self.redis_ins_name} 端口存活检查失败:{ins[0]}上存在{cluster_self_connet_port}")
+                    raise ValidationError(f"{self.redis_ins_name} 端口存活检查失败:{ins[0]}上存在{cluster_self_connet_port}")
+        return True
+
     def save_cluster_ins(self):
         """
         redis实例信息入库

polls/models.py

@@ -371,6 +371,9 @@ class RedisSentienlConf(models.Model):
                                              default="%s 1%")
     logfile = models.CharField(max_length=150, help_text="Redis日志存放路径",
                                verbose_name="logfile", default="/opt/repoll/")
+    # 默认Redis密码
+    authPass = models.CharField(max_length=150, help_text="当master服务设置了密码保护时slave服务连接master的密码",
+                                verbose_name="auth-pass", null=True, blank=True, default="qZr3pet")
 
     def __str__(self):
         return "Sentinel 配置成功"
@@ -444,7 +447,7 @@ class ApplyRedisText(models.Model):
     """用于DBA配置已审批通过的Redis实例"""
     # ipaddr = models.ForeignKey(Ipaddr, on_delete=models.CASCADE, null=True)
     redis_ins = models.ForeignKey(RedisIns, to_field="redis_ins_name", on_delete=models.CASCADE)
-    apply_text = models.TextField(max_length=250, verbose_name="实例详情",
+    apply_text = models.TextField(max_length=2400, verbose_name="实例详情",
                                   blank=True, null=True, help_text="具体规则如下: </br>"
                                                                    "1. standalone类型：</br>"
                                                                    "masterIp:masterPort:memSize(M)(例如：10.10.xx.xx:2048)</br>"