Implementing issue #4771 (Hashicorp Vault plugin)

Improvements for issue #3454 (option to disable variable resolving in tooltips)
apache · Jan 10, 2025 · bf6eb73 · bf6eb73
1 parent f42a1cb
commit bf6eb73
Show file tree

Hide file tree

Showing 14 changed files with 500 additions and 18 deletions.
diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml
@@ -769,6 +769,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-tech-vault</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-abort</artifactId>

diff --git a/core/src/main/java/org/apache/hop/core/variables/Variables.java b/core/src/main/java/org/apache/hop/core/variables/Variables.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.config.HopConfig;
 import org.apache.hop.core.exception.HopException;
@@ -36,6 +37,8 @@
 import org.apache.hop.metadata.api.IHopMetadataSerializer;
 import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
 import org.apache.hop.metadata.util.HopMetadataInstance;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
 
 /** This class is an implementation of IVariables */
 public class Variables implements IVariables {
@@ -151,7 +154,10 @@ public synchronized String resolve(String aString) {
 
     String resolved = StringUtil.environmentSubstitute(aString, properties);
 
-    resolved = substituteVariableResolvers(resolved);
+    String r = substituteVariableResolvers(resolved);
+    if (r != null) {
+      resolved = r;
+    }
 
     return resolved;
   }
@@ -164,7 +170,7 @@ private String substituteVariableResolvers(String input) {
       if (resolverIndex < 0) {
         // There's nothing more to do here.
         //
-        return resolved;
+        return null;
       }
 
       // Is there a close token?
@@ -173,21 +179,28 @@ private String substituteVariableResolvers(String input) {
       if (closeIndex < 0) {
         // False positive on the open token
         //
-        return resolved;
+        return null;
       }
 
-      // The variable resolver String [resolverIndex, closeIndex$]
-      // is in the format: #{name:argument}
-      //
+      /*
+       The following variable resolver string
+
+          [resolverIndex, closeIndex$]
+
+       is in the format:
+
+         #{name:arguments}
+
+      */
       int colonIndex = resolved.indexOf(':', resolverIndex);
       String name;
-      String argument;
+      String arguments;
       if (colonIndex >= 0) {
         name = resolved.substring(resolverIndex + StringUtil.RESOLVER_OPEN.length(), colonIndex);
-        argument = resolved.substring(colonIndex + 1, closeIndex);
+        arguments = resolved.substring(colonIndex + 1, closeIndex);
       } else {
         name = resolved.substring(resolverIndex + StringUtil.RESOLVER_OPEN.length(), closeIndex);
-        argument = "";
+        arguments = "";
       }
 
       try {
@@ -205,9 +218,55 @@ private String substituteVariableResolvers(String input) {
           }
           return resolved;
         }
-        String resolvedArgument = resolver.getIResolver().resolve(argument, this);
 
-        // We take the first part of the original resolved string, the resolved argument, and the
+        /*
+
+         The arguments come in the form: path-key:value-key
+         For example: #{vault:hop/data/some-db:hostname}
+         The secret path part will be "hop/data/some-db" and the secret value part is "hostname".
+
+        */
+        String secretPath;
+        String secretValue = null;
+
+        String[] parameters = arguments.split(":");
+        secretPath = parameters[0];
+        if (parameters.length > 1) {
+          secretValue = parameters[1];
+        }
+
+        String resolvedArgument = resolver.getIResolver().resolve(secretPath, this);
+        if (StringUtils.isEmpty(resolvedArgument)) {
+          return resolved;
+        }
+
+        // If we have a value to retrieve from the JSON we got back, we can do that:
+        //
+        if (StringUtils.isEmpty(secretValue)) {
+          return resolvedArgument;
+        } else {
+          try {
+            JSONObject js = (JSONObject) new JSONParser().parse(resolvedArgument);
+            Object value = js.get(secretValue);
+            if (value == null) {
+              // Value not found in the secret
+              resolvedArgument = "";
+            } else {
+              resolvedArgument = value.toString();
+            }
+          } catch (Exception e) {
+            LogChannel.GENERAL.logError(
+                "Error parsing JSON '"
+                    + resolvedArgument
+                    + "} to retrieve secret value '"
+                    + secretValue
+                    + "'",
+                e);
+            // Keep the origin String
+          }
+        }
+
+        // We take the first part of the original resolved string, the resolved arguments, and the
         // last part
         // to continue resolving all expressions in the string.
         //
@@ -217,7 +276,6 @@ private String substituteVariableResolvers(String input) {
 
         // Where do we continue?
         startIndex = before.length() + resolvedArgument.length();
-
       } catch (HopException e) {
         throw new RuntimeException(
             "Error resolving variable '" + input + "' with variable resolver metadata", e);
@@ -240,7 +298,7 @@ private String substituteVariableResolvers(String input) {
   @Override
   public String resolve(String aString, IRowMeta rowMeta, Object[] rowData)
       throws HopValueException {
-    if (aString == null || aString.length() == 0) {
+    if (aString == null || aString.isEmpty()) {
       return aString;
     }
 

diff --git a/core/src/main/java/org/apache/hop/core/variables/resolver/IVariableResolver.java b/core/src/main/java/org/apache/hop/core/variables/resolver/IVariableResolver.java
@@ -30,12 +30,12 @@ public interface IVariableResolver {
   /**
    * Resolve the variables in the given String.
    *
-   * @param string The String in which we want to replace variables.
+   * @param secretPath The
    * @param variables The variables and values to use as a reference
    * @return The input string with expressions resolved.
    * @throws HopException In case something goes wrong.
    */
-  String resolve(String string, IVariables variables) throws HopException;
+  String resolve(String secretPath, IVariables variables) throws HopException;
 
   void setPluginId();
 

diff --git a/.../main/java/org/apache/hop/core/variables/resolver/GooleSecretManagerVariableResolver.java b/.../main/java/org/apache/hop/core/variables/resolver/GooleSecretManagerVariableResolver.java
@@ -29,6 +29,7 @@
 import org.apache.hop.core.gui.plugin.GuiElementType;
 import org.apache.hop.core.gui.plugin.GuiPlugin;
 import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.logging.LogChannel;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.metadata.api.HopMetadataProperty;
 
@@ -77,7 +78,7 @@ public void init() {
   public String resolve(String secretId, IVariables variables) throws HopException {
 
     if (StringUtils.isEmpty(secretId)) {
-      return secretId;
+      return null;
     }
 
     try {
@@ -115,8 +116,9 @@ public String resolve(String secretId, IVariables variables) throws HopException
         return response.getPayload().getData().toStringUtf8();
       }
     } catch (Exception e) {
-      throw new HopException(
+      LogChannel.GENERAL.logError(
           "Error looking up secret key '" + secretId + "' in Google Secret Manager", e);
+      return null;
     }
   }
 

diff --git a/plugins/tech/pom.xml b/plugins/tech/pom.xml
@@ -37,5 +37,6 @@
         <module>google</module>
         <module>neo4j</module>
         <module>parquet</module>
+        <module>vault</module>
     </modules>
 </project>
diff --git a/plugins/tech/vault/pom.xml b/plugins/tech/vault/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.hop</groupId>
+        <artifactId>hop-plugins-tech</artifactId>
+        <version>2.12.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>hop-tech-vault</artifactId>
+    <packaging>jar</packaging>
+    <name>Hop Plugins Technology Hashicorp Vault</name>
+
+    <properties>
+        <vault.driver.version>2.0.0</vault.driver.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.bettercloud</groupId>
+            <artifactId>vault-java-driver</artifactId>
+            <version>${vault.driver.version}</version>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/plugins/tech/vault/src/assembly/assembly.xml b/plugins/tech/vault/src/assembly/assembly.xml
@@ -0,0 +1,56 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 http://maven.apache.org/xsd/assembly-2.2.0.xsd">
+    <id>hop-tech-parquet</id>
+    <formats>
+        <format>zip</format>
+    </formats>
+    <baseDirectory>.</baseDirectory>
+    <files>
+        <file>
+            <source>${project.basedir}/src/main/resources/version.xml</source>
+            <outputDirectory>plugins/tech/vault</outputDirectory>
+            <filtered>true</filtered>
+        </file>
+    </files>
+
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/src/main/samples</directory>
+            <outputDirectory>config/projects/samples/</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <includes>
+                <include>org.apache.hop:hop-tech-vault:jar</include>
+            </includes>
+            <outputDirectory>plugins/tech/vault</outputDirectory>
+        </dependencySet>
+        <dependencySet>
+            <scope>runtime</scope>
+            <includes>
+                <include>com.bettercloud:vault-java-driver:jar:</include>
+            </includes>
+            <outputDirectory>plugins/tech/vault/lib</outputDirectory>
+        </dependencySet>
+    </dependencySets>
+</assembly>