diff --git a/.gitignore b/.gitignore index ece2343af4..1e7ad161f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.iml .idea .DS_Store +*.class assembly/target @@ -20,4 +21,14 @@ plugins/azkaban/linkis-jobtype/target/ plugins/linkis/linkis-appjoint-entrance/target/ sendemail-appjoint/sendemail-core/target/ -visualis-appjoint/appjoint/target/ \ No newline at end of file +visualis-appjoint/appjoint/target/ + +dss-user-manager/target/ +logs +### Example user template template +### Example user template + +# IntelliJ project files + +out +gen diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..0a9b7a79be --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "editor.formatOnPaste": true, + "editor.formatOnType": true, + "editor.formatOnSave": true +} \ No newline at end of file diff --git a/assembly/pom.xml b/assembly/pom.xml index 3aa0e2d9a4..5395815964 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/conf/config.sh b/conf/config.sh index c6d910aca9..74c913ddc6 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -77,4 +77,4 @@ AZKABAN_ADRESS_PORT=8081 QUALITIS_ADRESS_IP=127.0.0.1 QUALITIS_ADRESS_PORT=8090 -DSS_VERSION=0.9.0 +DSS_VERSION=0.9.1 diff --git a/datachecker-appjoint/pom.xml b/datachecker-appjoint/pom.xml index 9d36a02ae5..d609c3ab32 100644 --- a/datachecker-appjoint/pom.xml +++ b/datachecker-appjoint/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md b/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md index 598ff158b8..d1b5408a2f 100644 --- a/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md +++ b/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md @@ -6,7 +6,7 @@ ```xml - 0.9.0 + 0.9.1 0.9.4 2.11.8 1.8 diff --git a/docs/en_US/ch2/DSS_0.9.1_upgrade_notes.md b/docs/en_US/ch2/DSS_0.9.1_upgrade_notes.md new file mode 100644 index 0000000000..f1261483f4 --- /dev/null +++ b/docs/en_US/ch2/DSS_0.9.1_upgrade_notes.md @@ -0,0 +1,85 @@ +# DSS 0.9.1 upgrade notes + +## Environmental description + +------ + +1. The user who installs the node machine deployment of DSS must have the permission to create the directory in hdfs + 1)If the hadoop cluster adopts the kerberos authentication mechanism, in order to prevent the ticket from expiring, you need to execute the knit -kt command on the client, for example:kinit -kt /etc/security/keytabs/hdfs.keytab yarn/xxxxxxxx + 2)If the hadoop cluster uses the simple authentication mechanism, use hdfs dfs chmod authorization, such as: hdfs dfs chmod 775 /user/hive/xx +2. The user who deploys DSS on the DSS node has the permission to create hive database: + 1)If hadoop cluster using simple authentication mechanism, you can try the following manner authorized: + hive>set system:user:name=dss; + hive> grant all to user dss; + +Currently, there is no automatic authorization in the script, and the user needs to execute the command manually. +2)If the Hadoop cluster adopts kerberos authentication, the kinit command is automatically executed in our script to obtain the ticket, and there is no need to execute the command manually. The user only needs to configure the kerberos related parameters. For the specific configuration, see the kerberos configuration chapter. + +The newly created user should be configured in hive.users.in.admin.role in hive-site.xml. + +1. LDAP must be installed (user authentication only supports LDAP), and there must be ou=user and ou=group entries in ldap, such as: ou=user,dc=baidu,dc=com和ou=group,dc=baidu,dc=com. + The ldap version supports 2.4.x, and the support of other versions is to be verified +2. Install sshpass service, yum -y install sshpass + +## Upgrade installation instructions + +------ + +The jar package involved in this change: Under the dss-server/lib directory:dss-application-0.9.1.jar,dss-server-0.9.1.jar,dss-user-manager-0.9.1.jar +Front-end static files: web +After replacing the above file with the latest one, then modify the following configuration file + +### Installation and configuration file instructions + +1. kerberos related + +Function description: If the Hadoop cluster uses the kerberos authentication mechanism, the newly created user will be granted kerberos permissions +Configuration file path: dss-server/conf/linkis.properties + +``` + Parameters: + wds.linkis.kerberos.enable.switch --Whether the cluster adopts the kerberos authentication mechanism, 0-do not use kerberos 1-use kerberos. If the Hadoop cluster does not use the kerberos authentication mechanism, none of the following parameters need to be configured. + wds.linkis.kerberos.keytab.path --The storage location of keytab on the DSS installation node can be arbitrarily specified, such as /etc/security/keytabs + wds.linkis.kerberos.kdc.node --Deploy the KDC service node IP, such as 192.168.1.1 + wds.linkis.kerberos.ssh.port --Deploy the KDC service node SSH port number, generally 22 + wds.linkis.kerberos.kdc.user.name --Deploy a linux user name on the KDC node, the user must have sudo permission (very important!!!) for ssh operation + wds.linkis.kerberos.kdc.user.password --The login password of the kdc node user mentioned above, used for ssh operation + wds.linkis.kerberos.realm --Kerberos manages the domain name of the hadoop cluster, please consult the cluster operation and maintenance personnel + wds.linkis.kerberos.admin--A user granted the admin role on kerberos (such as hdfs, very important!!!, otherwise the authorization cannot be completed) +``` + +1. metastore related + +Function description: create hive databases for newly created user and grant the newly created user permission +Parameter configuration file: dss-server/conf/linkis.properties + +``` +Parameters: + wds.linkis.metastore.hive.hdfs.base.path --The path where hive warehouse data is stored on hdfs, such as /user/hive/warehouse + wds.dss.deploy.path --dss_linkis installation package path, such as /usr/local/dss_linkis +``` + +1. ldap related + Function description: Create a new Entry under ou=user and ou=group of ldap for user login authentication + Parameter configuration file path: tools/bin/ldap_user.py + +``` +LDAP_HOST -- Install the ldap service IP, such as 192.168.11.11 +LDAP_BASEDN --The upper dn of ou=user or ou=group, such as dc=example,dc=com +LDAP_ADMIN -- The dn of the user logging in to the ldap service, such as cn=admin,dc=example,dc=cn +LDAP_PASS --Password for logging in to the ldap service +The first line in the ldap_user.py file #!/usr/local/tools/venv/bin/python, replace /user/local with the installation path of dss_linkis +``` + +## User manual + +------ + +1. Access address [http://url](http://url/):port/#/userManger, you need to login as a super user (installation user) +2. Server configuration: + If the hadoop cluster uses the simple authentication mechanism, the user needs to add the ip, login user name (with sudo permission), and password of each server in the yarn cluster.The underlying principle is that the server where dss is installed will ssh to each server in the yarn cluster, and then create a linux user. + +If the kerberos authentication mechanism adopted by the hadoop cluster, just add an ip (such as 127.0.0.1), username, and password. If not added, the interface will report an exception, and subsequent versions will fix this bug. + +1. WorkspaceRootPath, hdfsRootPath, resultRootPath, schedulerPath, DSS installation directory, Azkaban installation directory.The default value is consistent with the configuration in the installation configuration file config.sh. The directory can be either the hdfs directory, starting with hdfs:///, or the linux directory, starting with file:///. +2. The bottom layer will create a hive database for the user, the database name: xx_default, and give the permission to add, delete, modify, and select. \ No newline at end of file diff --git "a/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" "b/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" index 6ac3ece635..e89a29638d 100644 --- "a/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" @@ -6,7 +6,7 @@ ```xml - 0.9.0 + 0.9.1 0.9.4 2.11.8 1.8 diff --git "a/docs/zh_CN/ch2/DSS_0.9.1_\345\215\207\347\272\247\350\257\264\346\230\216.md" "b/docs/zh_CN/ch2/DSS_0.9.1_\345\215\207\347\272\247\350\257\264\346\230\216.md" new file mode 100644 index 0000000000..203b315f10 --- /dev/null +++ "b/docs/zh_CN/ch2/DSS_0.9.1_\345\215\207\347\272\247\350\257\264\346\230\216.md" @@ -0,0 +1,72 @@ +# DSS-0.9.1升级说明 + +## 环境说明 + +1. 安装DSS节点上部署DSS用户必须有hdfs创建目录的权限 + 1)如果hadoop集群采用kerberos认证机制，为防止票据过期，则需要在客户端执行 knit -kt命令,比如：kinit -kt /etc/security/keytabs/hdfs.keytab yarn/xxxxxxxx + 2)如果hadoop集群采用simple认证机制，则使用hdfs dfs chmod 授权，比如:hdfs dfs chmod 775 /user/hive/xx +2. 安装DSS节点上部署DSS的用户具有创建hive database权限问题: + 1)如果hadoop集群采用simple认证机制，可以尝试如下方式授权： + hive>set system:user:name=dss; + hive> grant all to user dss + 目前并未在脚本中自动授权，需要用户手动执行命令。 + 2)如果hadoop集群采用kerberos认证，在我们的脚本中自动执行了kinit命令以获取票据，不需要手工执行命令，用户只需要要配置kerberos相关的参数，具体配置见kerberos配置章节。 + 新建的用户要在 hive-site.xml 中hive.users.in.admin.role配置。 +3. 必须安装有LDAP（用户认证只支持LDAP），ldap中必须有ou=user和ou=group条目，比如：ou=user,dc=baidu,dc=com和ou=group,dc=baidu,dc=com。ldap版本支持2.4.x,其他版本的支持情况待验证 +4. 安装sshpass服务，yum -y install sshpass + +## 版本升级安装说明 + +本次改动涉及的的jar包:dss-server/lib目录下: dss-application-0.9.1.jar,dss-server-0.9.1.jar,dss-user-manager-0.9.1.jar +前端静态文件：web +将以上文件替换成最新的后，然后修改下面的配置文件 + +### 安装及配置文件说明 + +1. kerberos相关 + 功能说明：如果hadoop集群采用kerberos认证机制，则会给新建的用户授予kerberos权限 + 配置文件路径：dss-server/conf/linkis.properties + +``` + 参数： + wds.linkis.kerberos.enable.switch --集群是否采用kerberos认证机制，0-不采用kerberos 1-采用kerberos。如果hadoop集群不采用kerberos认证机制，则下面的参数都不需要配置。 + wds.linkis.kerberos.keytab.path --keytab在DSS安装节点上的存放位置，可以任意指定，比如 /etc/security/keytabs + wds.linkis.kerberos.kdc.node --部署KDC服务节点IP,比如192.168.1.1 + wds.linkis.kerberos.ssh.port --部署KDC服务节点SSH端口号，一般都是22 + wds.linkis.kerberos.kdc.user.name --部署KDC节点上的一个linux用户名,该用用户必须有sudo权限(重要，重要！！！)，用于ssh操作 + wds.linkis.kerberos.kdc.user.password --上面提到的kdc节点用户的登录密码，用于ssh操作 + wds.linkis.kerberos.realm --kerberos管理hadoop集群的域名，请咨询集群运维人员。 + wds.linkis.kerberos.admin--kerberos上的一个被授予admin角色的用户（如hdfs,非常重要！！！！，否则无法完成授权） +``` + +1. metastore相关 + 功能说明：给新建用户hive库，并授予新建用户权限 + 参数配置文件：dss-server/conf/linkis.properties + +``` + 参数： + wds.linkis.metastore.hive.hdfs.base.path --hive仓库数据存储在在hdfs上的路径，比如 /user/hive/warehouse + wds.dss.deploy.path --dss_linkis安装包路径，比如 /usr/local/dss_linkis +``` + +3.ldap相关 +功能说明：在ldap的ou=user和ou=group下新建一个Entry，用于用户登录验证 +参数配置文件路径：安装包下的tools/bin/ldap_user.py + +``` +LDAP_HOST -- 安装ldap服务IP，比如192.168.11.11 +LDAP_BASEDN --ou=user或ou=group的上层dn，比如 dc=example,dc=com +LDAP_ADMIN -- 登录ldap服务的用户dn 比如 'cn=admin,dc=example,dc=cn' +LDAP_PASS --登录ldap服务的密码 +ldap_user.py文件中第一行 #!/usr/local/tools/venv/bin/python,将/user/local换成dss_linkis的安装路径 +``` + +## 使用说明 + +1. 访问地址 http://url:port/#/userManger,需要用超级用户(安装用户)登录 +2. 服务器配置： + 如果hadoop集群采用的是simple认证机制，则用户需要添加yarn集群各服务器的ip、登录用户名（具有sudo权限）、密码。其底层的原理是安装dss的服务器会ssh到yarn集群的各服务器,然后创建linux用户。 + 如果hadoop集群采用的kerberos认证机制，则随便添加一个ip（比如127.0.0.1）,用户名,密码。如果不添加接口会报异常，后续版本会修复此bug +3. workspaceRootPath,hdfsRootPath,resultRootPath,schedulerPath,DSS安装目录，Azkaban安装目录。 + 其默认值和安装配置文件config.sh里配置的保持一致。其目录既可以是hdfs目录,以hdfs:///开头，也可以是linux目录，以file:///开头 +4. 底层会给用户创建hive库,库名：xx_default,并赋予增删改查权限 \ No newline at end of file diff --git a/dss-appjoint-auth/pom.xml b/dss-appjoint-auth/pom.xml index 941a120d1c..3a368b8666 100644 --- a/dss-appjoint-auth/pom.xml +++ b/dss-appjoint-auth/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/dss-appjoint-core/pom.xml b/dss-appjoint-core/pom.xml index 6ee707c4dd..7140b33656 100644 --- a/dss-appjoint-core/pom.xml +++ b/dss-appjoint-core/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/dss-appjoint-loader/pom.xml b/dss-appjoint-loader/pom.xml index b36ab5d799..99ab6ac8be 100644 --- a/dss-appjoint-loader/pom.xml +++ b/dss-appjoint-loader/pom.xml @@ -22,12 +22,12 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 dss-appjoint-loader - 0.9.0 + 0.9.1 diff --git a/dss-application/pom.xml b/dss-application/pom.xml index 54583b1239..8be0e04348 100644 --- a/dss-application/pom.xml +++ b/dss-application/pom.xml @@ -23,7 +23,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 dss-application @@ -47,6 +47,12 @@ dss-appjoint-loader ${dss.version} + + org.apache.htrace + htrace-core + 3.2.0-incubating + compile + diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java index 5eca714b6e..1194eb4271 100644 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java +++ b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java @@ -26,4 +26,27 @@ public class ApplicationConf { public static final CommonVars FAQ = CommonVars.apply("wds.linkis.application.dws.params",""); + + public static final String SUPER_USER_NAME = CommonVars.apply("wds.linkis.super.user.name","").getValue(); + public static final String WORKSPACE_USER_ROOT_PATH = CommonVars.apply("wds.linkis.workspace.user.root.path","").getValue(); + public static final String HDFS_USER_ROOT_PATH = CommonVars.apply("wds.linkis.hdfs.user.root.path","").getValue(); + public static final String RESULT_SET_ROOT_PATH = CommonVars.apply("wds.linkis.result.set.root.path","").getValue(); + public static final String WDS_SCHEDULER_PATH = CommonVars.apply("wds.linkis.scheduler.path","").getValue(); + public static final String WDS_USER_PATH = CommonVars.apply("wds.linkis.user.path","hdfs:///user").getValue(); + public static final String DSS_INSTALL_DIR = CommonVars.apply("wds.linkis.dss.install.dir","").getValue(); + public static final String AZKABAN_INSTALL_DIR = CommonVars.apply("wds.linkis.azkaban.install.dir","").getValue(); + + + + + + + + + + + + + + } diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java index 43323288a4..73dd36aca0 100644 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java +++ b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java @@ -25,6 +25,14 @@ public class DSSUser { private String username; private String name; private Boolean isFirstLogin; + private boolean isSuperUser = false; + + public boolean getIsSuperUser() { + return isSuperUser; + } + public void setIsSuperUser(boolean isSuperUser) { + this.isSuperUser = isSuperUser; + } public Long getId() { return id; diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java new file mode 100644 index 0000000000..4f2b258915 --- /dev/null +++ b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.application.entity; + +public class WorkSpacePath { + private String workspaceRootPath; + private String hdfsRootPath; + private String resultRootPath; + private String schedulerPath; + private String userPath; + + public String getUserPath() { + return userPath; + } + + public void setUserPath(String userPath) { + this.userPath = userPath; + } + + + + public String getWorkspaceRootPath() { + return workspaceRootPath; + } + + public void setWorkspaceRootPath(String workspaceRootPath) { + this.workspaceRootPath = workspaceRootPath; + } + + public String getHdfsRootPath() { + return hdfsRootPath; + } + + public void setHdfsRootPath(String hdfsRootPath) { + this.hdfsRootPath = hdfsRootPath; + } + + public String getResultRootPath() { + return resultRootPath; + } + + public void setResultRootPath(String resultRootPath) { + this.resultRootPath = resultRootPath; + } + + public String getSchedulerPath() { + return schedulerPath; + } + + public void setSchedulerPath(String schedulerPath) { + this.schedulerPath = schedulerPath; + } + + + + + + + + +} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java index 2842487a1c..10552b25f2 100644 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java +++ b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java @@ -17,9 +17,11 @@ package com.webank.wedatasphere.dss.application.restful; +import com.webank.wedatasphere.dss.application.conf.ApplicationConf; import com.webank.wedatasphere.dss.application.entity.Application; import com.webank.wedatasphere.dss.application.entity.DSSUser; import com.webank.wedatasphere.dss.application.entity.DSSUserVO; +import com.webank.wedatasphere.dss.application.entity.WorkSpacePath; import com.webank.wedatasphere.dss.application.handler.ApplicationHandlerChain; import com.webank.wedatasphere.dss.application.service.ApplicationService; import com.webank.wedatasphere.dss.application.service.DSSApplicationUserService; @@ -37,6 +39,8 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; /** @@ -70,7 +74,31 @@ public Response getBaseInfo(@Context HttpServletRequest req){ } DSSUser dssUser = dataworkisUserService.getUserByName(username); DSSUserVO dataworkisUserVO = new DSSUserVO(); + String superUserName = ApplicationConf.SUPER_USER_NAME; + if(username.equals(superUserName)){ + dssUser.setIsSuperUser(true); + }else{ + dssUser.setIsSuperUser(false); + } + dataworkisUserVO.setBasic(dssUser); return Message.messageToResponse(Message.ok().data("applications",applicationList).data("userInfo",dataworkisUserVO)); } + + + @GET + @Path("paths") + public Response getWorkSpace(@Context HttpServletRequest req) throws Exception { + WorkSpacePath workSpacePath = new WorkSpacePath(); + workSpacePath.setHdfsRootPath(ApplicationConf.HDFS_USER_ROOT_PATH); + workSpacePath.setResultRootPath(ApplicationConf.RESULT_SET_ROOT_PATH); + workSpacePath.setSchedulerPath(ApplicationConf.WDS_SCHEDULER_PATH); + workSpacePath.setWorkspaceRootPath(ApplicationConf.WORKSPACE_USER_ROOT_PATH); + workSpacePath.setUserPath(ApplicationConf.WDS_USER_PATH); + ArrayList> responses = ApplicationUtils.convertToMap(workSpacePath); + return Message.messageToResponse(Message.ok().data("paths",responses) + .data("dssInstallDir", ApplicationConf.DSS_INSTALL_DIR) + .data("azkakanDir", ApplicationConf.AZKABAN_INSTALL_DIR)); + + } } diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java index 4f0fabd1b9..af65bd3227 100644 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java +++ b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java @@ -17,12 +17,16 @@ package com.webank.wedatasphere.dss.application.util; +import com.webank.wedatasphere.dss.application.entity.WorkSpacePath; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; /** * Created by chaogefeng on 2019/11/20. @@ -46,8 +50,36 @@ public static String redirectUrlFormat(String redirectUrl,String url){ return String.format(REDIRECT_FORMAT,redirectUrl,URLEndoder(url)); } - public static void main(String[] args) throws DSSErrorException { - System.out.println(redirectUrlFormat("http://127.0..0.1:8090/qualitis/api/v1/redirect","http://127.0..0.1:8090/#/projects/list?id={projectId}&flow=true")); + + public static ArrayList> convertToMap(Object obj) + throws Exception { + + ArrayList> mapList = new ArrayList(); + Field[] fields = obj.getClass().getDeclaredFields(); + for (int i = 0, len = fields.length; i < len; i++) { + String varName = fields[i].getName(); + boolean accessFlag = fields[i].isAccessible(); + fields[i].setAccessible(true); + HashMap map = new HashMap<>(); + Object o = fields[i].get(obj); + if (o != null){ + map.put("key",varName); + map.put("value",o.toString()); + mapList.add(map); + } + + fields[i].setAccessible(accessFlag); + } + + return mapList; + } + + public static void main(String[] args) throws Exception { +// System.out.println(redirectUrlFormat("http://127.0..0.1:8090/qualitis/api/v1/redirect","http://127.0..0.1:8090/#/projects/list?id={projectId}&flow=true")); + + WorkSpacePath workSpacePath = new WorkSpacePath(); + workSpacePath.setWorkspaceRootPath("/"); + System.out.println(convertToMap(workSpacePath)); } } diff --git a/dss-azkaban-scheduler-appjoint/pom.xml b/dss-azkaban-scheduler-appjoint/pom.xml index f7a82d5557..4f67a636b8 100644 --- a/dss-azkaban-scheduler-appjoint/pom.xml +++ b/dss-azkaban-scheduler-appjoint/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java index ab9de4608f..b24fe94e76 100644 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java +++ b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java @@ -49,17 +49,28 @@ public final class AzkabanSecurityService extends AppJointUrlImpl implements Sch static { Utils.defaultScheduler().scheduleAtFixedRate(()->{ - LOGGER.info("开始读取用户token文件"); + LOGGER.info("load azkaban-user.xml"); Properties properties = new Properties(); try { properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME)); userToken = properties; } catch (IOException e) { - LOGGER.error("读取文件失败:",e); + LOGGER.error("load error:",e); } },0,10, TimeUnit.MINUTES); } + public void reloadToken(){ + LOGGER.info("reload azkaban-user.xml"); + Properties properties = new Properties(); + try { + properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME)); + userToken = properties; + } catch (IOException e) { + LOGGER.error("reload error:",e); + } + } + @Override public void setBaseUrl(String baseUrl) { this.securityUrl = baseUrl + "/checkin"; diff --git a/dss-common/pom.xml b/dss-common/pom.xml index fb77c4dad1..4fe637f11b 100644 --- a/dss-common/pom.xml +++ b/dss-common/pom.xml @@ -23,7 +23,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 dss-common diff --git a/dss-flow-execution-entrance/pom.xml b/dss-flow-execution-entrance/pom.xml index 7e152ced39..669ca30cd5 100644 --- a/dss-flow-execution-entrance/pom.xml +++ b/dss-flow-execution-entrance/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/dss-linkis-node-execution/pom.xml b/dss-linkis-node-execution/pom.xml index 0e8eb4e823..f773b12f24 100644 --- a/dss-linkis-node-execution/pom.xml +++ b/dss-linkis-node-execution/pom.xml @@ -24,7 +24,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 dss-linkis-node-execution diff --git a/dss-scheduler-appjoint-core/pom.xml b/dss-scheduler-appjoint-core/pom.xml index 979a90903e..72f112372a 100644 --- a/dss-scheduler-appjoint-core/pom.xml +++ b/dss-scheduler-appjoint-core/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java index 6113d3a5cc..a9652e52c4 100644 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java +++ b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java @@ -23,4 +23,5 @@ * Created by enjoyyin on 2019/10/12. */ public interface SchedulerSecurityService extends SecurityService { + void reloadToken(); } diff --git a/dss-server/pom.xml b/dss-server/pom.xml index c3ec0ed388..76d1446ddb 100644 --- a/dss-server/pom.xml +++ b/dss-server/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 @@ -93,6 +93,10 @@ gson com.google.code.gson + + jsr311-api + javax.ws.rs + @@ -145,7 +149,33 @@ com.webank.wedatasphere.dss dss-scheduler-appjoint-core - 0.9.0 + 0.9.1 + + + com.webank.wedatasphere.dss + dss-user-manager + 0.9.1 + compile + + + com.github.rholder + guava-retrying + 2.0.0 + + + dom4j + dom4j + 1.6.1 + + + com.typesafe + config + 1.4.1 + + + xml-apis + xml-apis + 1.4.01 diff --git a/dss-server/src/main/assembly/distribution.xml b/dss-server/src/main/assembly/distribution.xml index 0c7f789ee4..0da5c182ca 100644 --- a/dss-server/src/main/assembly/distribution.xml +++ b/dss-server/src/main/assembly/distribution.xml @@ -300,6 +300,24 @@ conf unix + + ${project.parent.basedir}/dss-user-manager/src/main/resources/config + + * + + 0777 + conf/config + unix + + + ${project.parent.basedir}/dss-user-manager/src/main/resources/default + + * + + 0777 + conf/default + unix + ${basedir}/bin diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java index e255b125f1..8a7344f9b2 100644 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java +++ b/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java @@ -161,7 +161,7 @@ private static void initDWCApplication() { } DWCException.setApplicationName(serviceInstance.getApplicationName()); DWCException.setHostname(Utils.getComputerName()); - DWCException.setHostPort(Integer.parseInt(applicationContext.getEnvironment().getProperty("server.port"))); + DWCException.setHostPort(Integer.parseInt(applicationContext.getEnvironment().getProperty("server.port","9004"))); } private static void setServiceInstance(ServiceInstance serviceInstance) throws Exception { diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java index 95ab683080..2c5711e154 100644 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java +++ b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java @@ -24,8 +24,10 @@ public class DSSServerConstant { public static final String DSS_PROJECT_SOURCE = "create by user"; public static final String DSS_WORKSPACE_SOURCE = "create by user"; public static final String PROJECT_VERSION_ID = "projectVersionID"; - public static final String PUBLISH_FLOW_REPORT_FORMATE = "工作流名:%s,版本号:%s，工作流内容为空,请自行修改或者删除"; + public static final String PUBLISH_FLOW_REPORT_FORMATE = "the workflow name is% s, the version number is% s, and the workflow content is empty. Please modify or delete it by yourself"; public static final String EMVEDDEDFLOWID = "\"embeddedFlowId\":"; public static final String VERSION_FORMAT = "%06d"; public static final String VERSION_PREFIX = "v"; + public static final String SUPER_USER_LOGIN_ERROR = "please login with super user"; + } diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java new file mode 100644 index 0000000000..a249e7ae3b --- /dev/null +++ b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java @@ -0,0 +1,92 @@ +package com.webank.wedatasphere.dss.server.restful; + +import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; +import com.webank.wedatasphere.dss.appjoint.scheduler.SchedulerAppJoint; +import com.webank.wedatasphere.dss.application.conf.ApplicationConf; +import com.webank.wedatasphere.dss.application.service.ApplicationService; +import com.webank.wedatasphere.dss.server.constant.DSSServerConstant; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.impl.UserAuthorizationClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * @program: user-authorization + * @description: 鲁班对外交互的接口 包括施工 注册用户 + * @create: 2020-08-12 14:24 + **/ + +@Component +@Path("/dss") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class UserManagerApi { + + private UserAuthorizationClient client = new UserAuthorizationClient(); + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private ApplicationService applicationService; + + private SchedulerAppJoint schedulerAppJoint; + + @POST + @Path("/user") + public Response createUser(@Context HttpServletRequest req, AuthorizationBody body) { + String username = SecurityFilter.getLoginUsername(req); + String superUserName = ApplicationConf.SUPER_USER_NAME; + if(!username.equals(superUserName)){ + return Message.messageToResponse(Message.error(DSSServerConstant.SUPER_USER_LOGIN_ERROR)); + } + + try { + String result = client.authorization(body); + + if(result.equals(AbsCommand.SUCCESS)){ + schedulerAppJoint = getSchedulerAppJoint(); + if(schedulerAppJoint != null){ + try{ + schedulerAppJoint.getSecurityService().reloadToken(); + }catch (Throwable throwable){ + logger.warn("choose schedulies,don not care"); + } + + } + return Message.messageToResponse(Message.ok()); + }else { + return Message.messageToResponse(Message.error(AbsCommand.ERROR)); + } + } catch (Exception e) { + return Message.messageToResponse(Message.error(e.getMessage())); + } + + + + + } + + private SchedulerAppJoint getSchedulerAppJoint(){ + + if(schedulerAppJoint == null){ + try { + schedulerAppJoint = (SchedulerAppJoint)applicationService.getAppjoint("schedulis"); + } catch (AppJointErrorException e) { + logger.error("Schedule system init failed!", e); + } + } + return schedulerAppJoint; + } + + +} diff --git a/dss-server/src/main/resources/default/AddschedulerUser.sh b/dss-server/src/main/resources/default/AddschedulerUser.sh new file mode 100644 index 0000000000..116c2d96b8 --- /dev/null +++ b/dss-server/src/main/resources/default/AddschedulerUser.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +user=$1 +password=$2 +installDir=$3 + + +if grep -i "^${user}=" $installDir/token.properties; + then + sed -i '' "s/^$user=.*/$user=$password/" $installDir/token.properties +else + echo "$user=$password" >> $installDir/token.properties +fi diff --git a/dss-server/src/main/resources/default/CreateLdapAccount.sh b/dss-server/src/main/resources/default/CreateLdapAccount.sh new file mode 100644 index 0000000000..1318ab16b7 --- /dev/null +++ b/dss-server/src/main/resources/default/CreateLdapAccount.sh @@ -0,0 +1,18 @@ +#!/bin/bash +source /etc/profile +server_host=$1 +server_login_user=$2 +server_login_password=$3 +server_python_path=$4 +server_ldap_source_path=$5 +ldap_user=$6 +ldap_password=$7 +echo "$server_login_password ssh $server_login_user@$server_host sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password" +sshpass -p $server_login_password ssh $server_login_user@$server_host "sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password" +#sshpass -p $server_login_password ssh $server_login_user@$server_host "sudo su - root -c 'source /etc/profile && source $server_ldap_source_path && sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password && deactivate'" + +echo "******************LDAP USER CREATED***********************" + + + + diff --git a/dss-server/src/main/resources/default/CreateLinuxUser.sh b/dss-server/src/main/resources/default/CreateLinuxUser.sh new file mode 100644 index 0000000000..fabcac1c5a --- /dev/null +++ b/dss-server/src/main/resources/default/CreateLinuxUser.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +source /etc/profile +server_user_strs=$1 +echo $server_user_strs +add_user_name=$2 +add_user_password=$3 +server_user_array=(${server_user_strs//,/ }) +for server_user_str in ${server_user_array[@]} +do + server_user_info=(${server_user_str//#/ }) + server_host=${server_user_info[0]} + server_user_name=${server_user_info[1]} + server_user_password=${server_user_info[2]} + echo "${server_host},${server_user_name},${server_user_password}" + + sudo sshpass -p $server_user_password ssh -o ConnectTimeout=1 $server_user_name@$server_host "echo success" + [ $? -ne 0 ] && echo "登录主机${server_host}失败" && exit 254 +done + +echo "************服务器网络校验通过,开始创建用户*****************" + +for server_user_str in ${server_user_array[@]} +do + + server_user_info=(${server_user_str//#/ }) + server_host=${server_user_info[0]} + server_user_name=${server_user_info[1]} + server_user_password=${server_user_info[2]} + + #sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name && echo $add_user_password |sudo -i passwd --stdin $add_user_name" + sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name -s /sbin/nologin" + + [ $? -ne 0 ] && echo "创建用户失败：${host}失败" && exit 254 +done diff --git a/dss-server/src/main/resources/default/HdfsPath.sh b/dss-server/src/main/resources/default/HdfsPath.sh new file mode 100644 index 0000000000..cd765486c5 --- /dev/null +++ b/dss-server/src/main/resources/default/HdfsPath.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +user=$1 +dir=$2 +hdfs dfs -mkdir -p $dir +hdfs dfs -chown $user:$user $dir \ No newline at end of file diff --git a/dss-server/src/main/resources/default/LinuxPath.sh b/dss-server/src/main/resources/default/LinuxPath.sh new file mode 100644 index 0000000000..bc55860b10 --- /dev/null +++ b/dss-server/src/main/resources/default/LinuxPath.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +user=$1 +dir=$2 +echo $1 $2; +sudo mkdir -p $dir +sudo chown $user:$user $dir \ No newline at end of file diff --git a/dss-server/src/main/resources/linkis.properties b/dss-server/src/main/resources/linkis.properties index 79b18225a5..97be7db018 100644 --- a/dss-server/src/main/resources/linkis.properties +++ b/dss-server/src/main/resources/linkis.properties @@ -17,7 +17,7 @@ wds.linkis.test.mode=true -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/ +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://0.0.0.1:3306/linkis?characterEncoding=UTF-8 wds.linkis.server.mybatis.datasource.username= @@ -38,9 +38,34 @@ wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.server. wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao ##azkaban -wds.dss.appjoint.scheduler.azkaban.address=http://127.0.0.1:8091 +wds.dss.appjoint.scheduler.azkaban.address=http://0.0.0.0:8081 wds.linkis.gateway.ip=127.0.0.1 wds.linkis.gateway.port=9001 wds.dss.appjoint.scheduler.project.store.dir=file:///appcom/tmp/wds/scheduler +wds.linkis.super.user.name=root +wds.linkis.workspace.user.root.path=file:///tmp/linkis/ +wds.linkis.hdfs.user.root.path=hdfs:///tmp/linkis +wds.linkis.result.set.root.path=hdfs:///tmp/linkis +wds.linkis.scheduler.path=file:///appcom/tmp/wds/scheduler +wds.linkis.user.path=hdfs:///user +wds.linkis.dss.install.dir=/usr/local/dss_linkis/dss/dss-server +wds.linkis.azkaban.install.dir=/usr/local/dss_linkis/azkaban + +wds.linkis.metastore.hive.hdfs.base.path=/user/hive/warehouse +wds.linkis.metastore.script.path=default/Metastore.sh +wds.linkis.metastore.db.tail=_default + +wds.linkis.kerberos.realm= +wds.linkis.kerberos.admin= +wds.linkis.kerberos.enable.switch=0 +wds.linkis.kerberos.script.path=default/Kerberos.sh +wds.linkis.kerberos.keytab.path=/etc/security/keytabs +wds.linkis.kerberos.kdc.node= +wds.linkis.kerberos.kdc.user.name= +wds.linkis.kerberos.kdc.user.password= +wds.linkis.kerberos.ssh.port=22 +wds.dss.deploy.path=/usr/local/dss_linkis +wds.dss.user.account.command.class=com.webank.wedatasphpere.dss.user.service.impl.LinuxUserCommand,com.webank.wedatasphpere.dss.user.service.impl.KerberosCommand,com.webank.wedatasphpere.dss.user.service.impl.LdapCommand,com.webank.wedatasphpere.dss.user.service.impl.WorkspaceCommand,com.webank.wedatasphpere.dss.user.service.impl.MetastoreCommand,com.webank.wedatasphpere.dss.user.service.impl.AzkabanCommand +wds.dss.scheduler.url=/schedule/system \ No newline at end of file diff --git a/dss-server/src/main/test/com/webank/TestUnit.java b/dss-server/src/main/test/com/webank/TestUnit.java new file mode 100644 index 0000000000..07c619ae98 --- /dev/null +++ b/dss-server/src/main/test/com/webank/TestUnit.java @@ -0,0 +1,14 @@ +package com.webank; + +import com.webank.wedatasphpere.dss.user.service.impl.UserAuthorizationClient; +import org.junit.Test; + +public class TestUnit { + + @Test + public void test() throws Exception { + + UserAuthorizationClient UserAuthorizationClient = new UserAuthorizationClient(); + + } +} diff --git a/dss-user-manager/pom.xml b/dss-user-manager/pom.xml new file mode 100644 index 0000000000..ef03b3a667 --- /dev/null +++ b/dss-user-manager/pom.xml @@ -0,0 +1,87 @@ + + + + dss + com.webank.wedatasphere.dss + 0.9.1 + + 4.0.0 + + dss-user-manager + jar + + UTF-8 + + + + + com.webank.wedatasphere.linkis + linkis-module + ${linkis.version} + provided + + + scala-xml_2.11 + org.scala-lang.modules + + + guava + com.google.guava + + + + + com.webank.wedatasphere.linkis + linkis-common + 0.9.4 + + + com.github.rholder + guava-retrying + 2.0.0 + provided + + + org.junit.jupiter + junit-jupiter + RELEASE + test + + + dom4j + dom4j + 1.6.1 + + + cn.hutool + hutool-all + 5.3.2 + + + mysql + mysql-connector-java + 5.1.49 + + + com.typesafe + config + 1.4.1 + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + \ No newline at end of file diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java new file mode 100644 index 0000000000..731194cc49 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +import java.util.ResourceBundle; + +/** + * @program: dss-appjoint-auth + * @description: 用户模块配置文件 + * + * @create: 2020-12-30 16:26 + **/ + + +public class DSSUserManagerConfig { +// private final static ResourceBundle resource = ResourceBundle.getBundle("linkis"); + public static final String LOCAL_USER_ROOT_PATH = CommonVars.apply("wds.dss.user.root.dir","null").getValue().trim(); + public static final String BDP_SERVER_MYBATIS_DATASOURCE_URL = CommonVars.apply("wds.linkis.server.mybatis.datasource.url", "null").getValue().trim(); + public static final String BDP_SERVER_MYBATIS_DATASOURCE_USERNAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.username", "null").getValue().trim(); + public static final String BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD = CommonVars.apply("wds.linkis.server.mybatis.datasource.password", "null").getValue().trim(); + public static final String SCHEDULER_ADDRESS = CommonVars.apply("wds.dss.appjoint.scheduler.azkaban.address", "null").getValue().trim(); + public static final String USER_ACCOUNT_COMMANDS = CommonVars.apply("wds.dss.user.account.command.class", "null").getValue().trim(); + + public static final String METASTORE_HDFS_PATH = CommonVars.apply("wds.linkis.metastore.hive.hdfs.base.path", "null").getValue().trim(); + public static final String METASTORE_SCRIPT_PAHT = CommonVars.apply("wds.linkis.metastore.script.path", "null").getValue().trim(); + public static final String METASTORE_DB_TAIL = CommonVars.apply("wds.linkis.metastore.db.tail", "_default").getValue().trim(); + + public static final String KERBEROS_REALM = CommonVars.apply("wds.linkis.kerberos.realm", "null").getValue().trim(); + public static final String KERBEROS_ADMIN = CommonVars.apply("wds.linkis.kerberos.admin", "null").getValue().trim(); + public static final String KERBEROS_SCRIPT_PATH = CommonVars.apply("wds.linkis.kerberos.script.path", "null").getValue().trim(); + public static final String KERBEROS_KEYTAB_PATH = CommonVars.apply("wds.linkis.kerberos.keytab.path", "null").getValue().trim(); + public static final String KERBEROS_SSH_PORT = CommonVars.apply("wds.linkis.kerberos.ssh.port", "22").getValue().trim(); + public static final String KERBEROS_KDC_NODE = CommonVars.apply("wds.linkis.kerberos.kdc.node", "null").getValue().trim(); + public static final String KERBEROS_KDC_USER_NAME = CommonVars.apply("wds.linkis.kerberos.kdc.user.name", "null").getValue().trim(); + public static final String KERBEROS_KDC_USER_PASSWORD = CommonVars.apply("wds.linkis.kerberos.kdc.user.password", "null").getValue().trim(); + public static final String KERBEROS_ENABLE_SWITCH = CommonVars.apply("wds.linkis.kerberos.enable.switch", "null").getValue().trim(); + public static final String DSS_DEPLOY_PATH = CommonVars.apply("wds.dss.deploy.path", "null").getValue().trim(); + public static final String DSS_SCHEDULER_URL = CommonVars.apply("wds.dss.scheduler.url", "/schedule/system").getValue().trim(); + + + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java new file mode 100644 index 0000000000..250f66f169 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.dto.request; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** + * @program: user-manager + * @description: 施工单数据结构 + * + * @create: 2020-08-12 14:29 + **/ +public class AuthorizationBody { + + private String username; + private String password; + private String dssInstallDir; + private String azkakanDir; + private ArrayList servers; + + + public ArrayList getServers() { + return servers; + } + + public void setServers(ArrayList servers) { + this.servers = servers; + } + + + + + public String getAzkakanDir() { + return azkakanDir; + } + + public void setAzkakanDir(String azkakanDir) { + this.azkakanDir = azkakanDir; + } + + public String getDssInstallDir() { + return dssInstallDir; + } + + public void setDssInstallDir(String installDir) { + this.dssInstallDir = installDir; + } + + public List> getPaths() { + return paths; + } + + public void setPaths(List> paths) { + this.paths = paths; + } + + private List> paths; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getDatabaseName(){ + return this.username + "_default"; + } + + + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java new file mode 100644 index 0000000000..7569612f5c --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.dto.request; + +public class LinuxServer { + + private String linuxHost; //master ip，Comma separated + private String linuxLoginUser; + private String linuxLoginPassword; + + + public String getLinuxHost() { + return linuxHost; + } + + public void setLinuxHost(String linuxHosts) { + this.linuxHost = linuxHosts; + } + + public String getLinuxLoginUser() { + return linuxLoginUser; + } + + public void setLinuxLoginUser(String linuxLoginUser) { + this.linuxLoginUser = linuxLoginUser; + } + + public String getLinuxLoginPassword() { + return linuxLoginPassword; + } + + public void setLinuxLoginPassword(String linuxLoginPassword) { + this.linuxLoginPassword = linuxLoginPassword; + } + + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java new file mode 100644 index 0000000000..be4bf3332f --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service; + + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import org.dom4j.DocumentException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.Response; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; + +/** + * 各模块的授权 继承这个类 根据需要实现自己的类。 + */ +public abstract class AbsCommand implements Command { + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public String capacity(AuthorizationBody body) { + return Command.SUCCESS; + } + + @Override + public String renew(AuthorizationBody body) { + return Command.SUCCESS; + } + + @Override + public String undoAuthorization(AuthorizationBody body) { return Command.SUCCESS; } + + @Override +// public String authorization(AuthorizationBody body) throws DocumentException { return Command.SUCCESS; } + public String authorization(AuthorizationBody body) throws IOException, Exception { return Command.SUCCESS; } + + public String toMessage(String msg) { + return this.getClass().getSimpleName() + "the module starts execution"+ msg; + } + + protected String runShell(String scriptPath, String[] args){ + String bashCommand; + try { + bashCommand = "sh " + scriptPath + " " + String.join(" ", args); + Runtime runtime = Runtime.getRuntime(); + Process process = runtime.exec(bashCommand); + + return this.getString(process); + } + catch (Exception e){ + logger.error(scriptPath, e); + return e.getMessage(); + } + } + + protected String getString(Process process) throws IOException, InterruptedException { + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + + String inline; + while ((inline = br.readLine()) != null) { + if (!inline.equals("")) { + inline = inline.replaceAll("<", "<").replaceAll(">", ">"); + logger.info(inline); + } else { + logger.info("\n"); + } + } + br.close(); + br = new BufferedReader(new InputStreamReader(process.getErrorStream())); //错误信息 + while ((inline = br.readLine()) != null) { + if (!inline.equals("")) + logger.warn(inline); + else + logger.warn("\n"); + } + + int status = process.waitFor(); + if (status != 0){ + logger.error("shell error: "+status); + } + br.close(); + return Command.SUCCESS; + } + + protected String getResource(String path){ + try { + URL url = this.getClass().getClassLoader().getResource(path); + return url.getPath(); + }catch (Exception e){ + logger.error("File does not exist " + path, e); + } + return null; + } +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java new file mode 100644 index 0000000000..dfed254468 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service; + + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import org.dom4j.DocumentException; + +import java.io.IOException; + +public interface Command { + + final public static String SUCCESS = "success"; + final public static String ERROR = "error"; + /** + * 授权开通服务 + * @param body + * @return 成功 success 其他失败 + */ + public String authorization(AuthorizationBody body) throws DocumentException, IOException, Exception; + + /** + * 关闭授权 + * @param body + * @return 成功 success 其他失败 + */ + public String undoAuthorization(AuthorizationBody body) throws Exception; + + /** + * 扩容 + * @param body + * @return 成功 success 其他失败 + */ + public String capacity(AuthorizationBody body) throws Exception; + + /** + * 续费 + * @param body + * @return 成功 success 其他失败 + */ + public String renew(AuthorizationBody body) throws Exception; +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java new file mode 100644 index 0000000000..0d2d3431cf --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service; + + +/** + * @program: user-authorization + * @description: 开通命令接口 + * + * @create: 2020-08-10 14:24 + **/ +public interface MacroCommand extends Command { + + public void add(AbsCommand command) throws Exception; + + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java new file mode 100644 index 0000000000..2f85ae38b3 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java @@ -0,0 +1,110 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.Element; +import org.dom4j.io.OutputFormat; +import org.dom4j.io.SAXReader; +import org.dom4j.io.XMLWriter; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +/** + * @program: dss-appjoint-auth + * @description: 开通azkaban账号 + * + * @create: 2021-01-08 15:53 + **/ + +public class AzkabanCommand extends AbsCommand { + @Override + public String authorization(AuthorizationBody body) { + + try{ + this.xmlHandler(body.getAzkakanDir()+"/conf/azkaban-users.xml", body); + String[] args = {body.getUsername(), body.getPassword(), body.getDssInstallDir()+"/conf/"}; + String path = getResource("default/AddschedulerUser.sh"); + return this.runShell(path, args); + }catch (Exception err){ + logger.error("AzkabanCommand auth error:", err); + return err.getMessage(); + } + } + + private void xmlHandler(String azkPath, AuthorizationBody body) throws DocumentException, IOException { + SAXReader reader = new SAXReader(); + + File file = new File(azkPath); + Document document; + FileInputStream fis = null; + try { + fis = new FileInputStream(file); + document = reader.read(fis); + }catch (DocumentException e){ + throw e; + }finally { + if (fis != null) { + fis.close(); + } + } + + Element root = document.getRootElement(); + + Iterator it = root.elementIterator("user"); + Boolean userExists = false; + Element element = null; + while (it.hasNext()) { + element = (Element) it.next(); + + String v = element.attributeValue("username"); + if(v.equals(body.getUsername())){ //修改密码 + userExists = true; + element.attribute("password").setValue(body.getPassword()); + } + } + if(!userExists){ //新增账号 + Element cloneEl = element.createCopy(); + cloneEl.attribute("username").setValue(body.getUsername()); + cloneEl.attribute("password").setValue(body.getPassword()); + + List elements = root.elements("user"); + elements.add(elements.size(), cloneEl); + } + + this.saveXml(document, file); + + } + + private void saveXml(Document document, File file) throws IOException { + FileOutputStream out =new FileOutputStream(file); + OutputFormat format=OutputFormat.createPrettyPrint(); + XMLWriter writer=new XMLWriter(out, format); + writer.write(document); + writer.close(); + } +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java new file mode 100644 index 0000000000..ef120f19e7 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; + +import java.io.IOException; +import java.net.InetAddress; + +/** + * @date 2021/1/5 + */ +public class KerberosCommand extends AbsCommand { + + @Override + public String authorization(AuthorizationBody body) throws Exception { + String rst = createKt(body); + return rst != Command.SUCCESS ? rst : Command.SUCCESS; + } + + private String createKt(AuthorizationBody body) throws Exception { + String userName = body.getUsername(); + String hostName = InetAddress.getLocalHost().getHostName(); + String res = null; + if(userName != null){ + res = callShell(DSSUserManagerConfig.KERBEROS_SCRIPT_PATH, userName,hostName, + DSSUserManagerConfig.KERBEROS_KEYTAB_PATH,DSSUserManagerConfig.KERBEROS_SSH_PORT, + DSSUserManagerConfig.KERBEROS_KDC_NODE,DSSUserManagerConfig.KERBEROS_KDC_USER_NAME,DSSUserManagerConfig.KERBEROS_KDC_USER_PASSWORD,DSSUserManagerConfig.KERBEROS_REALM,DSSUserManagerConfig.KERBEROS_ENABLE_SWITCH); + } + return res; + } + + private String callShell(String shellFile, String username, String hostName, String keytabPath, + String sshPort, String kdcNode, String kdcUser,String password, String realm,String enableSwich) throws Exception { + + String bashCommand = getResource(shellFile); + String scriptCmd ; + if(null != hostName){ + scriptCmd = String.format("%s %s %s %s %s %s %s %s %s", username,hostName,keytabPath,sshPort,kdcNode,kdcUser,password,realm,enableSwich); + }else { + scriptCmd = String.format("%s %s %s %s %s %s %s %s", username,keytabPath,sshPort,kdcNode,kdcUser,password,realm,enableSwich); + } + String[] args = scriptCmd.split(" "); + return this.runShell(bashCommand, args); + } + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java new file mode 100644 index 0000000000..31c2ecdbbf --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.List; + +/** + * @program: user-authorization + * @description: 创建用户空间 + * + * @create: 2020-08-13 13:39 + **/ + +public class LdapCommand extends AbsCommand { + + @Override + public String authorization(AuthorizationBody body) throws Exception { + + String userName = body.getUsername(); + String UserPassword = body.getPassword(); + String dssDeployPath = DSSUserManagerConfig.DSS_DEPLOY_PATH; + + String bashCommand = this.getClass().getClassLoader().getResource("default/CreateLdapAccount.sh").getPath(); + String[] args = { + userName, + UserPassword, + dssDeployPath + }; + + return this.runShell(bashCommand, args); + } + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java new file mode 100644 index 0000000000..10a567e216 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.dto.request.LinuxServer; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; + +import java.util.ArrayList; + +/** + * @program: user-authorization + * @description: 创建用户空间 + * + * @create: 2020-08-13 13:39 + **/ + +public class LinuxUserCommand extends AbsCommand { + + @Override + public String authorization(AuthorizationBody body) throws Exception { + +// String hosts = body.getLinuxHosts(); +// String linuxPassword = body.getLinuxLoginPassword(); +// String linuxUserName = body.getLinuxLoginUser(); + ArrayList linuxServers = body.getServers(); + logger.info("服务器ip"+linuxServers.toString()); + StringBuffer stringBuffer = new StringBuffer(); + for(LinuxServer linuxServer:linuxServers){ + String hosts = linuxServer.getLinuxHost(); + String linuxPassword = linuxServer.getLinuxLoginPassword(); + String linuxUserName = linuxServer.getLinuxLoginUser(); + stringBuffer.append(hosts).append("#").append(linuxUserName).append("#").append(linuxPassword).append(","); + } + stringBuffer.deleteCharAt(stringBuffer.lastIndexOf(",")); + String addUserName = body.getUsername(); + String addUserPassword = body.getPassword(); + String bashCommand = this.getClass().getClassLoader().getResource("default/CreateLinuxUser.sh").getPath(); + String[] args = {stringBuffer.toString(),addUserName,addUserPassword}; + + return this.runShell(bashCommand, args); + } + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java new file mode 100644 index 0000000000..6076919192 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @date 2021/1/5 + */ +public class MetastoreCommand extends AbsCommand { + private static final Logger logger = LoggerFactory.getLogger(MetastoreCommand.class); + + @Override + public String authorization(AuthorizationBody body) throws Exception { + String rst = createDb(body); + return rst != Command.SUCCESS ? rst : Command.SUCCESS; + } + + private String createDb(AuthorizationBody body) throws Exception { + String bashCommand = null; + String[] args = null; + String userName = body.getUsername(); + if (userName != null) { + String dbName = userName + DSSUserManagerConfig.METASTORE_DB_TAIL; + String path = DSSUserManagerConfig.METASTORE_HDFS_PATH + "/"+dbName+".db"; + bashCommand = getResource(DSSUserManagerConfig.METASTORE_SCRIPT_PAHT); + args = new String[]{ userName,dbName,path, + DSSUserManagerConfig.KERBEROS_REALM,DSSUserManagerConfig.KERBEROS_ADMIN,DSSUserManagerConfig.KERBEROS_KEYTAB_PATH,DSSUserManagerConfig.KERBEROS_ENABLE_SWITCH}; + } + return this.runShell(bashCommand, args); + } +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java new file mode 100644 index 0000000000..73351943b9 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java @@ -0,0 +1,146 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; + +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import cn.hutool.crypto.SecureUtil; +import cn.hutool.crypto.symmetric.DES; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +/** + * @date 2021/1/11 + */ +public class SchedulisCommand extends AbsCommand { + + private static final Logger logger = LoggerFactory.getLogger(SchedulisCommand.class); + + @Override + public String authorization(AuthorizationBody body) { + String username = body.getUsername(); + logger.info("开始新增调度用户：{}", username); + String password = body.getPassword(); + String encryptionPwd = getEncryptionPwd(username, password); + Connection connection; + Statement stmt; + try { + logger.info("开始插入ctyun_user表"); + connection = getConnection(); + stmt = connection.createStatement(); + String sql = "INSERT INTO `ctyun_user` (`id`,`name`,`username`,`email`,`password`,`work_order_item_config`) VALUES (?,?,?,?,?,NULL) ON DUPLICATE KEY UPDATE `password` = ?"; + PreparedStatement statement = connection.prepareCall(sql); + statement.setString(1, username); + statement.setString(2, username); + statement.setString(3, username); + statement.setString(4, username); + statement.setString(5, encryptionPwd); + statement.setString(6, encryptionPwd); + statement.executeUpdate(); + stmt.close(); + connection.close(); + logger.info("完成插入ctyun_user表"); + } catch (SQLException e) { + logger.error(e.getMessage()); + } + logger.info("开始调用接口新增schedulis用户"); + addSchedulisUser(username, password); + logger.info("结束调用接口新增schedulis用户"); + return Command.SUCCESS; + } + + private static Connection getConnection() { + try { + //注册数据库的驱动 + Class.forName("com.mysql.jdbc.Driver"); + //获取数据库连接（里面内容依次是：主机名和端口、用户名、密码） + String url = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_URL; + String user = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_USERNAME; + String password = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD; + return DriverManager.getConnection(url, user, password); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return null; + } + + private boolean addSchedulisUser(String username, String password) { + String schedulerUrl = DSSUserManagerConfig.DSS_SCHEDULER_URL; + RestTemplate restTemplate = new RestTemplate(); + String schedulisUrl = DSSUserManagerConfig.SCHEDULER_ADDRESS; + String url = new StringBuilder().append(schedulisUrl) + .append(schedulerUrl) + .toString(); + Map params = new HashMap<>(4); + params.put("userId", username); + params.put("password", password); + params.put("ajax", "addSystemUserViaFastTrackCtyun"); + String fullUrl = addParams(url, params); + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(fullUrl); + URI uri = builder.build().encode().toUri(); + ResponseEntity responseEntity = restTemplate.getForEntity(uri, JsonNode.class); + return HttpStatus.OK.equals(responseEntity.getStatusCode()); + } + + private static String addParams(String url, Map params) { + if (params.isEmpty()) { + return url; + } + StringBuilder sb = new StringBuilder().append(url).append("?"); + for (Map.Entry entry : params.entrySet()) { + if (StringUtils.hasText(entry.getValue())) { + sb.append(entry.getKey()) + .append("=") + .append(entry.getValue()) + .append("&"); + } + } + return sb.deleteCharAt(sb.length()-1).toString(); + } + + private String getEncryptionPwd(String username, String password) { + int minSize = 8; + while (username.length() < minSize) { + username += username; + } + byte[] keyBytes = username.getBytes(); + Arrays.sort(keyBytes); + DES des = SecureUtil.des(keyBytes); + return des.encryptHex(password); + } +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java new file mode 100644 index 0000000000..fd275a0e4b --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @program: user-authorization + * @description: + * + * @create: 2020-08-10 14:24 + **/ +public class UserAuthorizationClient { + + public UserMacroCommand userMacroCommand = new UserMacroCommand(); + protected final Logger logger = LoggerFactory.getLogger(UserAuthorizationClient.class); + + public UserAuthorizationClient() { + + String[] commandPaths = DSSUserManagerConfig.USER_ACCOUNT_COMMANDS.split(","); + for(String classPath: commandPaths){ + try { + userMacroCommand.add((AbsCommand) Class.forName(classPath).newInstance()); + } catch (Exception e) { + logger.info(e.getMessage()); + e.printStackTrace(); + } + } + } + + public String authorization(AuthorizationBody body) throws Exception { + return userMacroCommand.authorization(body); + } + + +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java new file mode 100644 index 0000000000..c842e666c3 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java @@ -0,0 +1,118 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.github.rholder.retry.*; +import com.google.common.base.Predicates; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; +import com.webank.wedatasphpere.dss.user.service.MacroCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + +/** + * @program: user-authorization + * @description: 开通命令实现 + * + * @create: 2020-08-10 14:24 + **/ +public class UserMacroCommand implements MacroCommand { + + private List commandList = new ArrayList<>(); + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + private static Integer RETRY_COUNT = 1; + + @Override + public void add(AbsCommand command) { + + commandList.add(command); + } + + @Override + public String authorization(AuthorizationBody body) throws Exception { + + return this.execute("authorization", body); + + } + + @Override + public String undoAuthorization(AuthorizationBody json) throws Exception { + + return this.execute("undoAuthorization", json); + + } + + @Override + public String capacity(AuthorizationBody json) throws Exception { + + return this.execute("capacity", json); + + } + + @Override + public String renew(AuthorizationBody json) throws Exception { + + return this.execute("renew", json); + + } + + + /** + * 授权操作基础方法 + * @param funName 调用的函数名 + * @param body 传入的数据 + * @return + * @throws ClassNotFoundException + * @throws NoSuchMethodException + */ + private String execute(String funName, AuthorizationBody body) throws Exception { + + for (AbsCommand command : commandList) { + + switch (funName) { + case "authorization": + command.authorization(body); + + case "undoAuthorization": + command.undoAuthorization(body); + + case "capacity": + command.capacity(body); + } + + } + + return "success"; + } + + + } + + + diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java new file mode 100644 index 0000000000..37b9f95972 --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; + +import java.util.HashMap; +import java.util.List; + +/** + * @program: user-authorization + * @description: 创建用户空间 + * + * @create: 2020-08-13 13:39 + **/ + +public class WorkspaceCommand extends AbsCommand { + + @Override + public String authorization(AuthorizationBody body) throws Exception { + List> paths = body.getPaths(); + String result = ""; + Boolean isSuccess = true; + for(HashMap map : paths){ + String path = map.get("value"); + String rst = createDir(path, body); + result += rst; + if(!rst.equals(Command.SUCCESS)){ + isSuccess = false; + } + } + if(isSuccess){ + return Command.SUCCESS; + } + logger.error(result); + return result; + } + + private String createDir(String path, AuthorizationBody body) throws Exception { + String bashCommand; + + if(path.contains("hdfs:")){ + path = path.replace("hdfs://", "") + "/" + body.getUsername(); + bashCommand = getResource("default/HdfsPath.sh"); + }else { + path = path.replace("file://", "") + "/" + body.getUsername(); + bashCommand = getResource("default/LinuxPath.sh"); + } + String[] args = {body.getUsername(), path}; + return this.runShell(bashCommand, args); + } +} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java new file mode 100644 index 0000000000..a4d854780a --- /dev/null +++ b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import com.webank.wedatasphpere.dss.user.service.AbsCommand; +import com.webank.wedatasphpere.dss.user.service.Command; + +/** + * @program: user-authorization + * @description: 开通yarn权限的实现 + * + * @create: 2020-08-10 14:24 + **/ +public class YarnCommand extends AbsCommand { + + @Override + public String authorization(AuthorizationBody body) { + + return Command.SUCCESS; + } +} diff --git a/dss-user-manager/src/main/resources/config/properties.conf b/dss-user-manager/src/main/resources/config/properties.conf new file mode 100644 index 0000000000..ece97f39a3 --- /dev/null +++ b/dss-user-manager/src/main/resources/config/properties.conf @@ -0,0 +1,14 @@ +#--Metastore配置 +base_path = #--元数据位于hdfs位置上的base +metastore_sh = default/Metastore.sh #--执行脚本的位置 +db_tail = #--hdfs上数据库文件的尾缀 +realm = #--kerberos域 +admin = #--获得hive metastore服务admin角色授权的用户（如hdfs,非常重要，否则无法完成授权） + +#--Kerberos配置 +shellFile = default/Kerberos.sh #--执行脚本的位置 +keytabPath = /etc/security/keytabs #--keytab在本节点上的存放位置 +sshPort = 22 #--ssh操作使用的端口 +kdcNode = #--kerberos服务kdc节点 +kdcUser = #--在进行ssh kdc操作是，对应kdc节点上的用户，应当知道该用户的密码，并且该用户应存在于kdc节点sudoers列表内 +password = "password" #--上面提到的kdc接应用户的登录密码，用于ssh操作 diff --git a/dss-user-manager/src/main/resources/default/AddschedulerUser.sh b/dss-user-manager/src/main/resources/default/AddschedulerUser.sh new file mode 100644 index 0000000000..d9a5a02e44 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/AddschedulerUser.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +user=$1 +password=$2 +installDir=$3 + + +if grep -i "^${user}=" $installDir/token.properties; + then + if [ "$(uname)" == "Darwin" ]; + then + sed -i '' "s/^$user=.*/$user=$password/" $installDir/token.properties + else + sed -i "s/^$user=.*/$user=$password/" $installDir/token.properties + fi +else + echo "$user=$password" >> $installDir/token.properties +fi diff --git a/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh b/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh new file mode 100644 index 0000000000..0f4c8cbe14 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh @@ -0,0 +1,7 @@ +#!/bin/bash +source /etc/profile +ldap_user=$1 +ldap_password=$2 +ldap_script_path=$3 +source $ldap_script_path/tools/venv/bin/activate && sudo $ldap_script_path/tools/venv/bin/python $ldap_script_path/tools/bin/ldap_user.py add_with_pw $ldap_user -p $ldap_password +echo "******************LDAP USER CREATED***********************" diff --git a/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh b/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh new file mode 100644 index 0000000000..32e42bccc0 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +source /etc/profile +server_user_strs=$1 +echo $server_user_strs +add_user_name=$2 +add_user_password=$3 +server_user_array=(${server_user_strs//,/ }) +for server_user_str in ${server_user_array[@]} +do + server_user_info=(${server_user_str//#/ }) + server_host=${server_user_info[0]} + server_user_name=${server_user_info[1]} + server_user_password=${server_user_info[2]} + echo "${server_host},${server_user_name},${server_user_password}" + + sudo sshpass -p $server_user_password ssh -o ConnectTimeout=1 $server_user_name@$server_host "echo success" + [ $? -ne 0 ] && echo "logon server:${server_host} failed" && exit 254 +done + +echo "************Start creating user*****************" + +for server_user_str in ${server_user_array[@]} +do + + server_user_info=(${server_user_str//#/ }) + server_host=${server_user_info[0]} + server_user_name=${server_user_info[1]} + server_user_password=${server_user_info[2]} + #sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name && echo $add_user_password |sudo -i passwd --stdin $add_user_name" + sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name -s /sbin/nologin" + + [ $? -ne 0 ] && echo "create user failed:${host}" && exit 254 +done diff --git a/dss-user-manager/src/main/resources/default/HdfsPath.sh b/dss-user-manager/src/main/resources/default/HdfsPath.sh new file mode 100644 index 0000000000..1f29565014 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/HdfsPath.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source /etc/profile +user=$1 +dir=$2 +echo $1 $2 +id $user +if [ $? -ne 0 ]; then + sudo useradd $user -s /sbin/nologin + echo "create user successfully" +fi + +hdfs dfs -mkdir -p $dir +hdfs dfs -chown $user:$user $dir diff --git a/dss-user-manager/src/main/resources/default/Kerberos.sh b/dss-user-manager/src/main/resources/default/Kerberos.sh new file mode 100644 index 0000000000..80c3eb502c --- /dev/null +++ b/dss-user-manager/src/main/resources/default/Kerberos.sh @@ -0,0 +1,146 @@ +#!/bin/bash +source /etc/profile +#需要将当前登录用户，如dss加入到sudoers + +#函数 +check_principal_exist(){ + all_principal=`timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"list_principals\""` #echo "all_principal:"$all_principal + principal=$1 + if [[ $all_principal =~ $principal ]] + then + #echo "包含" + return 1 + else + #echo "不包含" + return 0 + fi +} + +add_principal(){ + principalPrefix=$1 + echo "add_principal func,principalPrefix:"$principalPrefix + check_principal_exist "$principalPrefix@$REALM" + ifexist=$? + if [ $ifexist -eq 1 ] + then + echo "已有principal" + else + echo "没有principal,将会生成" + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"addprinc -randkey $principalPrefix\"" + fi +} + +generate_user(){ + username=$1 + if id -u $username >/dev/null 2>&1; then + echo "user exists" + else + echo "user does not exist, so we will create!" + sudo useradd $username + fi +} + +gen_keytab(){ + user=$1 + host=$2 + principalPrefix="$user/$host" + principal="$user/$host@$REALM" + add_principal $principalPrefix + if [[ $? -ne 0 ]];then + echo "create keytab failed!!!" + exit 1 + fi + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo rm -rf /tmp/$host.$user.keytab" + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"xst -norandkey -k /tmp/$host.$user.keytab $user/$host\"" + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo chmod 755 /tmp/$host.$user.keytab" + timeout 30 sshpass -p $PASSWORD scp -P $SSH_PORT $KDC_USER@$KDCSERVER:/tmp/$host.$user.keytab ./ + if [[ -f "$host.$user.keytab" ]]; then + sudo mv ./$host.$user.keytab $CENTER_KEYTAB_PATH/$user.keytab + if [[ $? != 0 ]];then + echo "rename keytab failed!" + else + generate_user $user + sudo chown $user $CENTER_KEYTAB_PATH/$user.keytab + sudo su - $user -c "kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal" + deployUser=`whoami` + sudo su - $deployUser -c "crontab -l > conf && echo '* */12 * * * sudo -u $user kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal' >> conf && crontab conf && rm -f conf" + fi + else + echo "the $user.keytab does not exist, please check your previous steps!" + fi +} + +gen_keytab_user(){ + user=$1 + principalPrefix="$user" + principal="$user@$REALM" + add_principal $principalPrefix + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo rm -rf /tmp/$user.keytab" + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"xst -norandkey -k /tmp/$user.keytab $user\"" + timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo chmod 755 /tmp/$user.keytab" + timeout 30 sshpass -p $PASSWORD scp -P $SSH_PORT $KDC_USER@$KDCSERVER:/tmp/$user.keytab ./ + if [[ -f "$user.keytab" ]]; then + sudo mv ./$user.keytab $CENTER_KEYTAB_PATH/$user.keytab + generate_user $user + sudo chown $user $CENTER_KEYTAB_PATH/$user.keytab + sudo su - $user -c "kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal" + sudo su - op -c "crontab -l > conf && echo '* */12 * * * sudo -u $user kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal' >> conf && crontab conf && rm -f conf" + else + echo "the $user.keytab does not exist, please check your previous steps!" + fi + +} + + + +#第一个参数为功能参数（必须有）,第二个为user（必须有），第三个为host（可以有） +if [ $# -lt 3 ] || [ $# -gt 9 ]; then + echo -e "\033[31m \033[05m请确认您的操作,输入格式如下 功能参数 [user|user hostname]\033[0m" + echo "Usage: $0 genenateKeytab {username|username hostname}" + echo `date '+%Y-%m-%d %H:%M:%S'`" parameters:"$* >>/tmp/deltaKerberos.log + exit 1 +else + if [ $# -eq 8 ]; then + user=$1 + CENTER_KEYTAB_PATH=$2 + SSH_PORT=$3 + KDCSERVER=$4 + KDC_USER=$5 + PASSWORD=$6 + REALM=$7 + KERBEROS_ENABLE=$8 + echo $user + echo $CENTER_KEYTAB_PATH + echo $SSH_PORT + echo $KDCSERVER_$KDC_USER + echo $REALM + echo $KERBEROS_ENABLE + if [ $KERBEROS_ENABLE = "0" ]; then + echo "kerberos is disabled" + else + echo "kerberos is enable" + echo `date '+%Y-%m-%d %H:%M:%S'`" in genenate_key_tab username:"$user >>/tmp/deltaKerberos.log + gen_keytab_user $user + fi + else + user=$1 + host=$2 + CENTER_KEYTAB_PATH=$3 + SSH_PORT=$4 + KDCSERVER=$5 + KDC_USER=$6 + PASSWORD=$7 + REALM=$8 + KERBEROS_ENABLE=$9 + echo $REALM + echo $KERBEROS_ENABLE + if [ $KERBEROS_ENABLE = "0" ]; then + echo "kerberos is disabled" + else + echo "kerberos1 is enable" + echo `date '+%Y-%m-%d %H:%M:%S'`" in genenate_key_tab username:"$user" hostname:"$host >>/tmp/deltaKerberos.log + gen_keytab $user $host + fi + fi +fi +exit 0 diff --git a/dss-user-manager/src/main/resources/default/LinuxPath.sh b/dss-user-manager/src/main/resources/default/LinuxPath.sh new file mode 100644 index 0000000000..03db38d857 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/LinuxPath.sh @@ -0,0 +1,12 @@ +#!/bin/bash +source /etc/profile +user=$1 +dir=$2 +echo $1 $2; +id $user +if [ $? -ne 0 ]; then + useradd $user -s /sbin/nologin + echo "create user successfully" +fi +sudo mkdir -p $dir +sudo chown $user:$user $dir diff --git a/dss-user-manager/src/main/resources/default/Metastore.sh b/dss-user-manager/src/main/resources/default/Metastore.sh new file mode 100644 index 0000000000..3d72f66589 --- /dev/null +++ b/dss-user-manager/src/main/resources/default/Metastore.sh @@ -0,0 +1,44 @@ +#!/bin/bash +source /etc/profile +user_name=$1 +db_name=$2 +path=$3 +realm=$4 +admin=$5 +ktPath=$6 +host_name=`hostname` +kerberos_enable=$7 +echo $kerberos_enable +if [ $kerberos_enable = "0" ]; then + echo "kerberos is disabled" +else + echo "kerberos is enabled" + kinit -kt $ktPath/$admin.keytab $admin/${host_name}@${realm} +fi + + #二、metastore操作 +hive -e "create database if not exists $db_name" +if [[ $? -ne 0 ]]; then + echo "create database failed!" +else + #修改数据库所属，将principal用户添加到metastore侧hive-site.xml hive.users.in.admin.role中 + if [ $kerberos_enable = "0" ]; then + hive -e "grant all on database $db_name to user $user_name" + else + hive -e "set role admin; grant all on database $db_name to user $user_name" + fi +fi + + #三、hdfs操作 +if [[ $? -ne 0 ]]; then + #回滚 + hive -e "drop database $db_name" + echo "grant database failed,rollback finished!" +else + echo "grant database $db_name successfully!" + #修改hdfs路径所属 + hdfs dfs -chown $user_name:$user_name $path + #修改hdfs路径权限 + hdfs dfs -chmod -R 700 $path + echo "hdfs operation successfully!" +fi diff --git a/dss-user-manager/src/main/resources/read_anlexander.txt b/dss-user-manager/src/main/resources/read_anlexander.txt new file mode 100644 index 0000000000..fc60487e09 --- /dev/null +++ b/dss-user-manager/src/main/resources/read_anlexander.txt @@ -0,0 +1,20 @@ +一键化开通账户 + 服务目录结构： + 进入口调用： + 入口调用：dss-user-manager\src\main\java\com\webank\wedatasphpere\dss\user\service\impl\UserAuthorizationClient.java + +#参考配置文件resource/config/properties.conf +#metastore功能使用说明 + 参数：base_path --元数据位于hdfs位置上的base + metastore_sh --执行脚本的位置 + db_tail --hdfs上数据库文件的尾缀 + realm --kerberos域 + admin --获得hive metastore服务admin角色授权的用户（如hdfs,非常重要，否则无法完成授权） + +#kerberos功能使用说明 + 参数：keytabPath --keytab在本节点上的存放位置 + shellFile --执行脚本的位置 + sshPort --ssh操作使用的端口 + kdcNode --kerberos服务kdc节点 + kdcUser --在进行ssh kdc操作是，对应kdc节点上的用户，应当知道该用户的密码，并且该用户应存在于kdc节点sudoers列表内 + password --上面提到的kdc接应用户的登录密码，用于ssh操作 \ No newline at end of file diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java new file mode 100644 index 0000000000..445b4889d4 --- /dev/null +++ b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +class KerberosCommandTest { + + @Test +// @MethodSource("body") + void authorization() { + AuthorizationBody body = new AuthorizationBody(); + body.setUsername("anlexander"); + body.setPassword("123321"); + KerberosCommand test = new KerberosCommand(); + try { + test.authorization(body); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("当前测试方法结束"); + } +} diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java new file mode 100644 index 0000000000..da1fcb91e2 --- /dev/null +++ b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import org.junit.jupiter.api.Test; + +class MetastoreCommandTest { + + @Test +// @MethodSource("body") + void authorization() { + AuthorizationBody body = new AuthorizationBody(); + body.setUsername("anlexander"); + body.setPassword("123321"); + MetastoreCommand test = new MetastoreCommand(); + + try { + test.authorization(body); + } catch (Exception e) { + e.printStackTrace(); + } + + System.out.println("当前测试方法结束"); + } +} diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java new file mode 100644 index 0000000000..68098eaaf4 --- /dev/null +++ b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package com.webank.wedatasphpere.dss.user.service.impl; + + +import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; +import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +class WorkspaceCommandTest { + + @Test +// @MethodSource("body") + void authorization() { + AuthorizationBody body = new AuthorizationBody(); + body.setUsername("luxl"); + body.setPassword("123321"); + WorkspaceCommand test = new WorkspaceCommand(); + try { + test.authorization(body); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("当前测试方法结束"); + +// DSSUserManagerConfig.LOCAL_USER_ROOT_PATH.getValue(); + } +} diff --git a/eventchecker-appjoint/pom.xml b/eventchecker-appjoint/pom.xml index 3ae6e8f70b..bdf035e3db 100644 --- a/eventchecker-appjoint/pom.xml +++ b/eventchecker-appjoint/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/plugins/azkaban/linkis-jobtype/pom.xml b/plugins/azkaban/linkis-jobtype/pom.xml index c1c697d8ca..9d0d787b5a 100644 --- a/plugins/azkaban/linkis-jobtype/pom.xml +++ b/plugins/azkaban/linkis-jobtype/pom.xml @@ -23,7 +23,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 com.webank.wedatasphere.dss linkis-jobtype diff --git a/plugins/linkis/linkis-appjoint-entrance/pom.xml b/plugins/linkis/linkis-appjoint-entrance/pom.xml index 12a064427e..bb7db19018 100644 --- a/plugins/linkis/linkis-appjoint-entrance/pom.xml +++ b/plugins/linkis/linkis-appjoint-entrance/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/pom.xml b/pom.xml index e80674603e..53346c0ffc 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ pom com.webank.wedatasphere.dss dss - 0.9.0 + 0.9.1 dss-common @@ -44,10 +44,11 @@ plugins/azkaban/linkis-jobtype plugins/linkis/linkis-appjoint-entrance assembly + dss-user-manager - 0.9.0 + 0.9.1 0.9.4 2.11.8 1.8 diff --git a/qualitis-appjoint/appjoint/pom.xml b/qualitis-appjoint/appjoint/pom.xml index 280e567a85..330b5cfd2f 100644 --- a/qualitis-appjoint/appjoint/pom.xml +++ b/qualitis-appjoint/appjoint/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/sendemail-appjoint/sendemail-core/pom.xml b/sendemail-appjoint/sendemail-core/pom.xml index 31440bd86c..3739647f0a 100644 --- a/sendemail-appjoint/sendemail-core/pom.xml +++ b/sendemail-appjoint/sendemail-core/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/visualis-appjoint/appjoint/pom.xml b/visualis-appjoint/appjoint/pom.xml index 11506add21..e572638daf 100644 --- a/visualis-appjoint/appjoint/pom.xml +++ b/visualis-appjoint/appjoint/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 0.9.0 + 0.9.1 4.0.0 diff --git a/web/package.json b/web/package.json index 5ac9b99f5e..f0f6dd8695 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dataspherestudio", - "version": "0.9.0", + "version": "0.9.1", "private": true, "scripts": { "serve": "vue-cli-service serve", diff --git a/web/src/commonData/i18n/common/en.json b/web/src/commonData/i18n/common/en.json index 0bb80defb2..02a3bf13e7 100644 --- a/web/src/commonData/i18n/common/en.json +++ b/web/src/commonData/i18n/common/en.json @@ -634,7 +634,8 @@ "navMune": { "FAQ": "FAQ", "clearCache": "Clear cache", - "logOut": "Log out" + "logOut": "Log out", + "userManager": "User Manager" }, "shape": { "dataDev": "Data development", @@ -1662,6 +1663,40 @@ "success": { "update": "Successfully updated global variables!" } + }, + "userManager": { + "createUser": "create user", + "username" : "username", + "password": "password", + "rootPath": "Root Path", + "spacePath": "Space root Path", + "hdfsPath": "HDFS root Path", + "resultPath": "Result root Path", + "schedulerPath": "Scheduler root Path", + "usernameNotNull": "username cannot be empty", + "usernameFormat": "username begins with a letter, contains only letters, numbers, or underscores", + "passwordNotNull": "password cannot be empty", + "passwordNotWeak": "password begins with a letter, contains letters, numbers, and special characters, not less than 8 digits", + "createSuccess": "create user Success", + "createFail": "create user Fail", + "dssInstallDir": "dss install directory", + "azkabanInstallDir": "Azkaban install directory", + "linuxHost" : "linuxHost", + "linuxLoginUser" : "linuxLoginUser", + "linuxLoginPassword" : "linuxLoginPassword", + "linuxHostNotNull" : "linuxHost cannot be empty", + "linuxLoginUserNotNull" : "linuxLoginUser cannot be empty", + "linuxLoginPasswordNotNull" : "linuxLoginPassword cannot be empty", + "serverSettings": "serverSettings", + "userSettings": "userSettings", + "XYB": "Next step", + "SYB": "Previous step", + "addServer": "Add server", + "deleteTip": "Are you sure you want to delete this server?", + "serverSame": " server's host and user is the same, please keep one server with the same configuration", + "serverNull": "The server configuration option cannot be empty", + "pathNotNull": "Cannot be empty", + "createTip": "It will take some time to create, please wait patiently, if successful, return to the home page, fail to stay on this page" } } } \ No newline at end of file diff --git a/web/src/commonData/i18n/common/zh.json b/web/src/commonData/i18n/common/zh.json index 3bd655377a..98d0946695 100644 --- a/web/src/commonData/i18n/common/zh.json +++ b/web/src/commonData/i18n/common/zh.json @@ -634,7 +634,8 @@ "navMune": { "FAQ": "常见问题", "clearCache": "清理缓存", - "logOut": "退出登录" + "logOut": "退出登录", + "userManager": "用户管理" }, "shape": { "dataDev": "数据开发", @@ -1662,6 +1663,40 @@ "success": { "update": "全局变量更新成功！" } + }, + "userManager": { + "createUser": "创建用户", + "username" : "账号", + "password": "密码", + "rootPath": "用户根目录", + "spacePath": "工作空间目录", + "hdfsPath": "HDFS根目录", + "resultPath": "结果存储根目录", + "schedulerPath": "调度存储根目录", + "usernameNotNull": "用户名不能为空", + "usernameFormat": "账号以字母开头，仅能含有字母、数字或者下划线", + "passwordNotNull": "密码不能为空", + "passwordNotWeak": "密码以字母开头，含有字母、数字和特殊字符，不少于8位", + "createSuccess": "创建账号成功", + "createFail": "创建账号失败", + "dssInstallDir": "DSS 安装目录", + "azkabanInstallDir": "Azkaban 安装目录", + "linuxHost" : "服务器ip", + "linuxLoginUser" : "服务器用户名", + "linuxLoginPassword" : "服务器密码", + "linuxHostNotNull" : "服务器ip不能为空", + "linuxLoginUserNotNull" : "服务器用户名不能为空", + "linuxLoginPasswordNotNull" : "服务器密码不能为空", + "serverSettings": "服务器配置", + "userSettings": "用户配置", + "XYB": "下一步", + "SYB": "上一步", + "addServer": "添加服务器", + "deleteTip": "确认删除该服务器?", + "serverSame": " 服务器ip与用户名配置相同，相同配置的服务器只能保留一个", + "serverNull": "服务器配置选项不能为空", + "pathNotNull": "不能为空", + "createTip": "创建需要一段时间，请耐心等候，如果成功则返回首页，失败则停留本页" } } } \ No newline at end of file diff --git a/web/src/js/module/header/index.vue b/web/src/js/module/header/index.vue index 8026c12ce9..b1dccf94d6 100644 --- a/web/src/js/module/header/index.vue +++ b/web/src/js/module/header/index.vue @@ -71,6 +71,7 @@ class="user-icon"/>
import('../view/commonIframe/index.vue'), }, + { + name: 'USER MANAGER', + path: 'userManager', + component: () => import('../view/userManager/index.vue'), + meta: { + title: 'userManager', + publicPage: true, + }, + }, { path: 'console', name: 'Console', diff --git a/web/src/js/view/userManager/index.vue b/web/src/js/view/userManager/index.vue new file mode 100644 index 0000000000..a56e2f6974 --- /dev/null +++ b/web/src/js/view/userManager/index.vue @@ -0,0 +1,375 @@ + + diff --git a/web/vue.config.js b/web/vue.config.js index 17750079eb..558b04caea 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -29,9 +29,10 @@ const getVersion = () => { return pkg.version; } -const host = "0.0.0.0"; -const port = "9001"; - +// const host = "0.0.0.0"; +// const port = "9001"; +const host = "***REMOVED***"; +const port = "8088"; module.exports = { publicPath: './', outputDir: 'dist/dist', @@ -80,6 +81,7 @@ module.exports = { } }, configureWebpack: { + devtool: process.env.NODE_ENV === 'development' ? 'eval-source-map' : '', resolve: { alias: { 'vue$': 'vue/dist/vue.esm.js',