4739 jdbc metadata cache enhanced

go.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export HOP_PATH=/Users/sergio/Serasoft/code/xdelox/hop
+# export HOP_DEPLOY=/Users/sergio/Serasoft/hops/2.12.0-SNAPSHOT/hop
+export HOP_DEPLOY=/Users/sergio/Serasoft/code/xdelox/hop/assemblies/client/target/hop
+
+# Check if an argument was provided
+if [ -n "$1" ] && [ "$1" = "rebuild" ]; then
+  echo "Building..."
+  mvn clean install -pl org.apache.hop:hop-core -DskipTests
+  mvn clean install  -pl org.apache.hop:hop-ui -DskipTests
+  mvn clean install  -pl org.apache.hop:hop-engine -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-textfile -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-excel -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-selectvalues -DskipTests
+ # mvn package -o -pl org.apache.hop:hop-transform-tableinput -DskipTests
+  mvn clean install -o -pl org.apache.hop:hop-transform-tableinput -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-misc-static-schema -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-javascript -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-script -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-databasejoin -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-transform-sql -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-action-sql -DskipTests
+#  mvn clean install -o -pl org.apache.hop:hop-action-waitforsql -DskipTests
+  echo $HOP_PATH
+  echo $HOP_DEPLOY
+
+ mv $HOP_PATH/core/target/hop-core-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+ mv $HOP_PATH/ui/target/hop-ui-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+ mv $HOP_PATH/engine/target/hop-engine-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+# mv $HOP_PATH/plugins/transforms/textfile/target/hop-transform-textfile-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+# mv $HOP_PATH/plugins/transforms/textfile/target/hop-transform-excel-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+ # mv $HOP_PATH/plugins/misc/static-schema/target/hop-misc-static-schema-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+# mv $HOP_PATH/plugins/transforms/selectvalues/target/hop-transform-selectvalues-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+mv $HOP_PATH/plugins/transforms/tableinput/target/hop-transform-tableinput-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+#  mv $HOP_PATH/plugins/transforms/janino/target/hop-transform-janino-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+#  mv $HOP_PATH/plugins/transforms/javascript/target/hop-transform-javascript-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+#  mv $HOP_PATH/plugins/transforms/sql/target/hop-transform-sql-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+#  mv $HOP_PATH/plugins/actions/sql/target/hop-action-sql-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+#  mv $HOP_PATH/plugins/actions/sql/target/hop-action-waitforsql-2.12.0-SNAPSHOT.zip $HOP_DEPLOY
+  cd $HOP_DEPLOY
+  unzip -o hop-core-2.12.0-SNAPSHOT.zip
+  unzip -o hop-ui-2.12.0-SNAPSHOT.zip
+  unzip -o hop-engine-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-textfile-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-excel-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-selectvalues-2.12.0-SNAPSHOT.zip
+  unzip -o hop-transform-tableinput-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-misc-static-schema-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-janino-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-javascript-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-script-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-databasejoin-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-transform-sql-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-action-sql-2.12.0-SNAPSHOT.zip
+#  unzip -o hop-action-waitforsql-2.12.0-SNAPSHOT.zip
+
+#  rm hop-transform-selectvalues-2.12.0-SNAPSHOT.zip
+  rm hop-transform-tableinput-2.12.0-SNAPSHOT.zip
+  rm hop-core-2.12.0-SNAPSHOT.zip
+  rm hop-ui-2.12.0-SNAPSHOT.zip
+  rm hop-engine-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-textfile-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-excel-2.12.0-SNAPSHOT.zip
+#  rm hop-misc-static-schema-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-janino-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-javascript-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-script-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-databasejoin-2.12.0-SNAPSHOT.zip
+#  rm hop-transform-sql-2.12.0-SNAPSHOT.zip
+#  rm hop-action-sql-2.12.0-SNAPSHOT.zip
+#  rm hop-action-waitforsql-2.12.0-SNAPSHOT.zip
+else
+    echo "Do nothing for now"
+fi
+
+
+$HOP_DEPLOY/hop-gui.sh debug

plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinDialog.java

@@ -23,6 +23,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.Props;
 import org.apache.hop.core.database.Database;
@@ -47,6 +51,7 @@
 import org.apache.hop.ui.core.widget.StyledTextComp;
 import org.apache.hop.ui.core.widget.TableView;
 import org.apache.hop.ui.core.widget.TextComposite;
+import org.apache.hop.ui.core.widget.highlight.SqlHighlight;
 import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
 import org.apache.hop.ui.util.EnvironmentUtils;
 import org.eclipse.swt.SWT;
@@ -63,6 +68,7 @@
 import org.eclipse.swt.layout.FormData;
 import org.eclipse.swt.layout.FormLayout;
 import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Display;
 import org.eclipse.swt.widgets.Label;
 import org.eclipse.swt.widgets.Shell;
 import org.eclipse.swt.widgets.TableItem;
@@ -96,13 +102,17 @@ public class DatabaseJoinDialog extends BaseTransformDialog {
   private Label wlCacheSize;
   private Text wCacheSize;
 
+  private AtomicReference<SqlHighlight> sqlHighlightListener =
+      new AtomicReference<>(new SqlHighlight(List.of()));
+
   public DatabaseJoinDialog(
       Shell parent,
       IVariables variables,
       DatabaseJoinMeta transformMeta,
       PipelineMeta pipelineMeta) {
     super(parent, variables, transformMeta, pipelineMeta);
     input = transformMeta;
+    initReservedWordsCache(pipelineMeta);
   }
 
   @Override
@@ -147,7 +157,11 @@ public String open() {
 
     // Connection line
     wConnection = addConnectionLine(shell, wTransformName, input.getConnection(), lsMod);
-    wConnection.addListener(SWT.Selection, e -> getSqlReservedWords());
+    wConnection.addListener(
+        SWT.Selection,
+        e -> {
+          onConnectionSelected();
+        });
 
     // ICache?
     Label wlCache = new Label(shell, SWT.RIGHT);
@@ -208,7 +222,6 @@ public void widgetSelected(SelectionEvent e) {
                 variables, shell, SWT.MULTI | SWT.LEFT | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL)
             : new SQLStyledTextComp(
                 variables, shell, SWT.MULTI | SWT.LEFT | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL);
-    wSql.addLineStyleListener(getSqlReservedWords());
     PropsUi.setLook(wSql, Props.WIDGET_STYLE_FIXED);
     wSql.addModifyListener(lsMod);
     FormData fdSql = new FormData();
@@ -418,6 +431,7 @@ public void widgetSelected(SelectionEvent e) {
     wCancel.addListener(SWT.Selection, e -> cancel());
 
     getData();
+    initSQLFormat();
 
     BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
 
@@ -586,4 +600,105 @@ private void get() {
           ke);
     }
   }
+
+  private void initSQLFormat() {
+    String connectionName = input.getConnection();
+    final Optional<List<String>> keywordsByConnectionName =
+        input.getKeywordsByConnectionName(connectionName);
+    if (keywordsByConnectionName.isPresent()) {
+      Display.getDefault()
+          .asyncExec(() -> refreshLineStyleListener(keywordsByConnectionName.get()));
+    } else {
+      refreshLineStyleListener(List.of());
+      Executors.newSingleThreadExecutor()
+          .submit(
+              () -> {
+                fetchKeywords(input.getConnection());
+                final Optional<List<String>> optKeywords =
+                    input.getKeywordsByConnectionName(connectionName);
+                if (optKeywords.isPresent()) {
+                  Display.getDefault().asyncExec(() -> refreshLineStyleListener(optKeywords.get()));
+                }
+              });
+    }
+  }
+
+  private void onConnectionSelected() {
+    if (input.containsKeywordsByConnectionName(wConnection.getText())) {
+      final List<String> k =
+          input.getKeywordsByConnectionName(wConnection.getText()).orElse(List.of());
+      refreshLineStyleListener(k);
+    } else {
+      final String currentConnection = wConnection.getText();
+      refreshLineStyleListener(List.of());
+      Executors.newSingleThreadExecutor()
+          .submit(
+              () -> {
+                fetchKeywords(currentConnection);
+                final List<String> k =
+                    input.getKeywordsByConnectionName(currentConnection).orElse(List.of());
+                Display.getDefault().asyncExec(() -> refreshLineStyleListener(k));
+              });
+    }
+  }
+
+  private synchronized void refreshLineStyleListener(List<String> keywords) {
+    wSql.removeLineStyleListener(sqlHighlightListener.get());
+    sqlHighlightListener.set(new SqlHighlight(keywords));
+    wSql.addLineStyleListener(sqlHighlightListener.get());
+    wSql.setText(wSql.getText());
+    // wSql.setRedraw(true);
+  }
+
+  private void initReservedWordsCache(PipelineMeta pipelineMeta) {
+    final ExecutorService executorService = Executors.newSingleThreadExecutor();
+    pipelineMeta.getDatabases().stream()
+        .forEach(
+            db -> {
+              executorService.submit(() -> fetchKeywords(db));
+            });
+  }
+
+  private void fetchKeywords(String connectionName) {
+    DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connectionName, variables);
+    if (databaseMeta != null) {
+      fetchKeywords(databaseMeta);
+    } else {
+      logError("Unable to find database '" + connectionName + "'");
+    }
+  }
+
+  private void fetchKeywords(DatabaseMeta databaseMeta) {
+
+    if (databaseMeta == null) {
+      logError("Database connection not found. Proceding without keywords.");
+      return;
+    }
+
+    try (Database db = new Database(loggingObject, variables, databaseMeta)) {
+      db.connect();
+      DatabaseMetaData databaseMetaData = db.getDatabaseMetaData();
+      if (databaseMetaData == null) {
+        logError("Couldn't get database metadata");
+        return;
+      }
+      List<String> sqlKeywords = new ArrayList<>();
+      try {
+        final ResultSet functionsResultSet = databaseMetaData.getFunctions(null, null, null);
+        while (functionsResultSet.next()) {
+          String functionName = functionsResultSet.getString("FUNCTION_NAME");
+          if (functionName.contains(";")) {
+            functionName = functionName.substring(0, functionName.indexOf(";"));
+          }
+          sqlKeywords.add(functionName);
+        }
+        sqlKeywords.addAll(Arrays.asList(databaseMetaData.getSQLKeywords().split(",")));
+      } catch (SQLException e) {
+        logError("Couldn't extract keywords from database metadata. Proceding without them.");
+      }
+      input.putKeywords(databaseMeta.getName(), sqlKeywords);
+    } catch (HopDatabaseException e) {
+      logError("Couldn't extract keywords from database metadata. Proceding without them.");
+    }
+  }
 }

plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinMeta.java

@@ -19,6 +19,9 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.hop.core.CheckResult;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.ICheckResult;
@@ -113,6 +116,8 @@ public class DatabaseJoinMeta extends BaseTransformMeta<DatabaseJoin, DatabaseJo
       injectionKeyDescription = "DatabaseJoinMeta.Injection.ReplaceVariables")
   private boolean replaceVariables;
 
+  private Map<String, List<String>> keywordsCache = new ConcurrentHashMap<>();
+
   public DatabaseJoinMeta() {
     super(); // allocate BaseTransformMeta
   }
@@ -592,4 +597,21 @@ public List<ParameterField> getParameters() {
   public void setParameters(List<ParameterField> parameters) {
     this.parameters = parameters;
   }
+
+  public void putKeywords(String name, List<String> sqlKeywords) {
+    keywordsCache.put(name, sqlKeywords);
+  }
+
+  public Map<String, List<String>> getKeywordsCache() {
+    return keywordsCache;
+  }
+
+  public Optional<List<String>> getKeywordsByConnectionName(String connection) {
+    final List<String> keywordsList = keywordsCache.get(connection);
+    return Optional.ofNullable(keywordsList);
+  }
+
+  public boolean containsKeywordsByConnectionName(String connection) {
+    return keywordsCache.containsKey(connection);
+  }
 }