From 6ed33462a2a9f26bca8bc1cf9688065729bbe5a9 Mon Sep 17 00:00:00 2001 From: Arunan Date: Sun, 2 Jan 2022 23:32:32 +0530 Subject: [PATCH 1/4] Support hot deployment in Service Catalog --- .../AppDeployerServiceComponent.java | 22 --------- .../application/deployer/CappDeployer.java | 26 ++++++++++ .../utils/ServiceCatalogUtils.java | 38 ++++++++++++++- .../test/ServiceCatalogTestCase.java | 47 +++++++++++++++---- 4 files changed, 102 insertions(+), 31 deletions(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java index 7b7af54355..623e118d77 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java @@ -30,7 +30,6 @@ import org.osgi.service.component.annotations.ReferenceCardinality; import org.osgi.service.component.annotations.ReferencePolicy; import org.wso2.carbon.securevault.SecretCallbackHandlerService; -import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.application.deployer.handler.DefaultAppDeployer; import org.wso2.micro.core.CarbonAxisConfigurator; import org.wso2.micro.integrator.dataservices.core.DBDeployer; @@ -39,18 +38,10 @@ import org.wso2.micro.integrator.initializer.deployment.application.deployer.CappDeployer; import org.wso2.micro.integrator.initializer.deployment.synapse.deployer.FileRegistryResourceDeployer; import org.wso2.micro.integrator.initializer.deployment.synapse.deployer.SynapseAppDeployer; -import org.wso2.micro.integrator.initializer.serviceCatalog.ServiceCatalogExecutor; import org.wso2.micro.integrator.initializer.services.SynapseEnvironmentService; import org.wso2.micro.integrator.initializer.utils.ConfigurationHolder; import org.wso2.micro.integrator.ndatasource.capp.deployer.DataSourceCappDeployer; -import java.util.ArrayList; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.wso2.micro.integrator.initializer.utils.Constants.ENABLE; -import static org.wso2.micro.integrator.initializer.utils.Constants.SERVICE_CATALOG_CONFIG; @Component(name = "org.wso2.micro.integrator.initializer.deployment.AppDeployerServiceComponent", immediate = true) public class AppDeployerServiceComponent { @@ -211,19 +202,6 @@ private void invokeRegisteredDeployers() { AxisConfigurator axisConfigurator = configCtx.getAxisConfiguration().getConfigurator(); if (axisConfigurator instanceof CarbonAxisConfigurator) { ((CarbonAxisConfigurator) axisConfigurator).deployServices(); - Map catalogProperties; - if (ConfigParser.getParsedConfigs().get(SERVICE_CATALOG_CONFIG) != null) { - catalogProperties = - (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( - SERVICE_CATALOG_CONFIG)).get(0); - if ((boolean) catalogProperties.get(ENABLE)) { - String repoLocation = ((CarbonAxisConfigurator) axisConfigurator).getRepoLocation(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.submit(new ServiceCatalogExecutor(repoLocation, secretCallbackHandlerService)); - // we are not interested in the result of this thread execution - executorService.shutdown(); - } - } } } diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java index 909f3a6eba..f95ce0b749 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java @@ -38,10 +38,13 @@ import org.wso2.micro.application.deployer.config.ApplicationConfiguration; import org.wso2.micro.application.deployer.config.Artifact; import org.wso2.micro.application.deployer.handler.AppDeploymentHandler; +import org.wso2.micro.core.CarbonAxisConfigurator; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.core.util.FileManipulator; import org.wso2.micro.integrator.initializer.dashboard.ArtifactDeploymentListener; +import org.wso2.micro.integrator.initializer.serviceCatalog.ServiceCatalogDeployer; import org.wso2.micro.integrator.initializer.utils.DeployerUtil; +import org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils; import java.io.File; import java.io.FileInputStream; @@ -55,6 +58,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.xml.namespace.QName; import javax.xml.stream.XMLStreamException; @@ -86,6 +91,16 @@ public class CappDeployer extends AbstractDeployer { */ private String extension; + /** + * Service Catalog Executor threads for publishing Services to Service Catalog. + */ + private ExecutorService serviceCatalogExecutor; + + /** + * Map object to store Service Catalog configuration + */ + private Map serviceCatalogConfiguration; + public void init(ConfigurationContext configurationContext) { if (log.isDebugEnabled()) { log.debug("Initializing Capp Deployer.."); @@ -96,6 +111,11 @@ public void init(ConfigurationContext configurationContext) { String appUnzipDir = AppDeployerUtils.getAppUnzipDir() + File.separator + AppDeployerUtils.getTenantIdString(); FileManipulator.deleteDir(appUnzipDir); + + if (ServiceCatalogUtils.isServiceCatalogEnabled()) { + serviceCatalogExecutor = Executors.newFixedThreadPool(10); + serviceCatalogConfiguration = ServiceCatalogUtils.readConfiguration(); + } } public void setDirectory(String cAppDir) { @@ -193,6 +213,12 @@ private void deployCarbonApps(String artifactPath) throws CarbonException { faultyCAppObjects.add(currentApp); faultyCapps.add(cAppName); } + if (ServiceCatalogUtils.isServiceCatalogEnabled()) { + ServiceCatalogDeployer serviceDeployer = new ServiceCatalogDeployer(cAppName, + ((CarbonAxisConfigurator) axisConfig.getAxisConfiguration().getConfigurator()).getRepoLocation(), + serviceCatalogConfiguration); + serviceCatalogExecutor.execute(serviceDeployer); + } } /** diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java index e6e0070b2e..0281c0af25 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java @@ -401,6 +401,29 @@ public static Map readConfiguration(SecretCallbackHandlerService return configMap; } + /** + * Read APIM host configurations from deployment.toml file. + * + * @param secretCallbackHandlerService secret callback handler reference. + * @return map of resolved values. + */ + public static Map readConfiguration() { + Map configMap = new HashMap<>(); + Map catalogProperties = + (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( + SERVICE_CATALOG_CONFIG)).get(0); + + String apimHost = catalogProperties.get(APIM_HOST); + + String userName = catalogProperties.get(USER_NAME); + String password = catalogProperties.get(PASSWORD); + + configMap.put(APIM_HOST, apimHost); + configMap.put(USER_NAME, userName); + configMap.put(PASSWORD, password); + return configMap; + } + /** * Process metadata folder and move to temporary location. * @@ -571,7 +594,7 @@ public static boolean archiveDir(String destArchiveName, String sourceDir) { if (alreadyUploaded) { log.info("Service catalog already contains the latest configs, aborting the service-catalog uploader"); } else { - log.info("Could not find metadata to upload, aborting the service-catalog uploader"); + log.info("Metadata not included, hence not publishing to Service Catalog"); } return false; } @@ -867,4 +890,17 @@ private static String getProxyServiceUrlFromMetadata(File yamlFile) { } return currentServiceUrl; } + + public static boolean isServiceCatalogEnabled() { + Map catalogProperties; + if (ConfigParser.getParsedConfigs().get(SERVICE_CATALOG_CONFIG) != null) { + catalogProperties = + (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( + SERVICE_CATALOG_CONFIG)).get(0); + if ((boolean) catalogProperties.get(ENABLE)) { + return true; + } + } + return false; + } } diff --git a/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java b/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java index 847e4f4a61..4bdc63cea6 100644 --- a/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java +++ b/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java @@ -65,6 +65,7 @@ public class ServiceCatalogTestCase extends ESBIntegrationTest { private static final String FAULTY_CAPP = "invalidCompositeApplication_1.0.0.car"; private static final String CAPP_WITH_META_AND_ENV = "blaCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String CAPP_WITH_PROXY_META = "proxyCompositeExporter_1.0.0-SNAPSHOT.car"; + private static final String CAPP_WITHOUT_META = "HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String NEW_CAPP_NAME = "demoCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String MODIFIED_NEW_CAPP_NAME = "changed_demoCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String SH_FILE_NAME = "micro-integrator.sh"; @@ -144,12 +145,42 @@ public void testServiceCatalogWithoutFaultyCAPP() serverConfigurationManager.removeFromCarbonapps(FAULTY_CAPP); serverConfigurationManager.restartMicroIntegrator(); assertTrue(Utils.checkForLog(carbonLogReader, - "Could not find metadata to upload, aborting the service-catalog uploader", 10), + "Metadata not included, hence not publishing to Service Catalog", 10), "Did not receive the expected info log"); } @Test(groups = {"wso2.esb"}, - description = "Test service catalog without setting env variables", priority = 3) + description = "Test service catalog by hot deploying CAapp without Metadata)", priority = 3) + public void testServiceCatalogHotDeploymentWithoutMetaData() + throws IOException, URISyntaxException, AutomationUtilException, InterruptedException { + carbonLogReader.clearLogs(); + File metadataCAPP = new File( + getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + + CAPP_WITHOUT_META); + serverConfigurationManager.copyToCarbonapps(metadataCAPP); + assertTrue(Utils.checkForLog(carbonLogReader, + "Metadata not included, hence not publishing to Service Catalog", 20), + "Did not receive the expected info log"); + serverConfigurationManager.removeFromCarbonapps(CAPP_WITHOUT_META); + } + + @Test(groups = {"wso2.esb"}, + description = "Test service catalog by hot deploying CAapp with Metadata)", priority = 4) + public void testServiceCatalogHotDeploymentWithMetaData() + throws IOException, URISyntaxException, AutomationUtilException, InterruptedException { + carbonLogReader.clearLogs(); + File metadataCAPP = new File( + getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + + CAPP_WITH_META_AND_ENV); + serverConfigurationManager.copyToCarbonapps(metadataCAPP); + assertTrue(Utils.checkForLog(carbonLogReader, + "Successfully updated the service catalog", 20), + "Did not receive the expected info log"); + serverConfigurationManager.removeFromCarbonapps(CAPP_WITH_META_AND_ENV); + } + + @Test(groups = {"wso2.esb"}, + description = "Test service catalog without setting env variables", priority = 5) public void testServiceCatalogMetadataWithoutEnv() throws IOException, AutomationUtilException, InterruptedException { File metadataCAPP = new File( @@ -162,7 +193,7 @@ public void testServiceCatalogMetadataWithoutEnv() } @Test(groups = {"wso2.esb"}, - description = "Test service catalog after setting env variables", priority = 4) + description = "Test service catalog after setting env variables", priority = 6) public void testServiceCatalogMetadataWithEnv() throws IOException, AutomationUtilException, InterruptedException { @@ -186,7 +217,7 @@ public void testServiceCatalogMetadataWithEnv() } @Test(groups = {"wso2.esb"}, - description = "Test the ZIP file created by the service catalog", priority = 5) + description = "Test the ZIP file created by the service catalog", priority = 7) public void testServiceCatalogZipFile() throws CarbonException, FileNotFoundException { File extracted = chekAndExtractPayloadZip(); assertTrue(extracted.exists(), "Error occurred while extracting the ZIP"); @@ -202,7 +233,7 @@ public void testServiceCatalogZipFile() throws CarbonException, FileNotFoundExce } @Test(groups = {"wso2.esb"}, - description = "Test MI is uploading only newly added APIs", priority = 6) + description = "Test MI is uploading only newly added APIs", priority = 8) public void testUploadOnlyNewAPIs() throws CarbonException, IOException, AutomationUtilException, InterruptedException { File newCAPP = new File( @@ -224,7 +255,7 @@ public void testUploadOnlyNewAPIs() } @Test(groups = {"wso2.esb"}, - description = "Test MI is uploading only modified APIs", priority = 7) + description = "Test MI is uploading only modified APIs", priority = 9) public void testUploadOnlyModifiedAPIs() throws CarbonException, IOException, AutomationUtilException, URISyntaxException, InterruptedException { // remove CAPP and add the modified one @@ -248,7 +279,7 @@ public void testUploadOnlyModifiedAPIs() } @Test(groups = {"wso2.esb"}, - description = "Test restart MI without any CAPP changes", priority = 8) + description = "Test restart MI without any CAPP changes", priority = 10) public void testMIRestart() throws IOException, AutomationUtilException, InterruptedException { serverConfigurationManager.applyMIConfigurationWithRestart(new File( getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + "FourthAPI" + @@ -262,7 +293,7 @@ public void testMIRestart() throws IOException, AutomationUtilException, Interru assertFalse(zipFile.exists(), "Payload.zip file should not be created"); } - @Test(groups = {"wso2.esb"}, description = "Test service catalog with proxy services", priority = 9) + @Test(groups = {"wso2.esb"}, description = "Test service catalog with proxy services", priority = 11) public void testServiceCatalogProxyServiceMetadata() throws CarbonException, IOException, AutomationUtilException, InterruptedException { File metadataCAPP = new File(getESBResourceLocation() + File.separator From 552c3dcf20eacc6e62063d1afdc983b2811b6c49 Mon Sep 17 00:00:00 2001 From: Arunan Date: Sun, 2 Jan 2022 23:53:12 +0530 Subject: [PATCH 2/4] Add missed files --- .../ServiceCatalogDeployer.java | 79 ++++++++++++++++++ ...tadataCompositeExporter_1.0.0-SNAPSHOT.car | Bin 0 -> 2785 bytes 2 files changed, 79 insertions(+) create mode 100644 components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java create mode 100644 integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java new file mode 100644 index 0000000000..b9733df90b --- /dev/null +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.micro.integrator.initializer.serviceCatalog; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.File; +import java.util.Map; + +import static org.wso2.micro.integrator.initializer.utils.Constants.*; +import static org.wso2.micro.integrator.initializer.utils.Constants.SERVICE_CATALOG; +import static org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils.*; +import static org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils.publishToAPIM; + +public class ServiceCatalogDeployer implements Runnable { + + private String cAppName; + private Map serviceCatalogConfiguration; + private final String repoLocation; + + private static final String CAPP_UNZIP_DIR; + + private static final Log log = LogFactory.getLog(ServiceCatalogDeployer.class); + + public ServiceCatalogDeployer(String name, String repoLocation, Map serviceCatalogConfiguration) { + this.cAppName = name; + this.repoLocation = repoLocation; + this.serviceCatalogConfiguration = serviceCatalogConfiguration; + } + + static { + String javaTmpDir = System.getProperty("java.io.tmpdir"); + CAPP_UNZIP_DIR = javaTmpDir.endsWith(File.separator) ? javaTmpDir + SERVICE_CATALOG : + javaTmpDir + File.separator + SERVICE_CATALOG; + } + + @Override + public void run() { + log.info("Executing Service Catalog deployer for CApp : " + cAppName); + + // check pre-conditions + if (!checkPreConditions()) return; + + // call service catalog and get all services + Map md5MapOfAllService = getAllServices(serviceCatalogConfiguration); + if (md5MapOfAllService == null) return; // error occurred while getting all services + + // create temporary directory to hold metadata. + if (!createTemporaryFolders(CAPP_UNZIP_DIR)) return; + + File tempDir = new File(CAPP_UNZIP_DIR, TEMP_FOLDER_NAME); + + // extract CAPPs and copy metadata to temp directory. + if (!extractMetadataFromCAPPs(tempDir, repoLocation, md5MapOfAllService)) return; + + // create the payload.zip file with extracted metadata + if (!archiveDir(CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME, tempDir.getPath())) return; + + // publish to service catalog endpoint. + publishToAPIM(serviceCatalogConfiguration, CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME); + } +} diff --git a/integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car b/integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car new file mode 100644 index 0000000000000000000000000000000000000000..30b95e4ad462db4365415b64eee8050c67fa3cf8 GIT binary patch literal 2785 zcmWIWW@Zs#;Nak3kld*h!hi%6fNYP{oSgjd{Gyzc@XV5o{L&KN)RIJ(#FE5#Lp=jM z1AS(O0B?2dAEI0=?kBKo^~1ARPTtFPnf(mGV}J7 z5BnnZSuf zzf6q&nyDXrm?3dh>h*~_pSq=mg6`9o-Dclb(Ct4>_BUVh*HS6@X$H9r_k-q4-7ogV zS738Wp|oxAWw*YyyNo)|pnU7XpvA^)q(ebVML6>fW>JoSZS$dU;$>FJX;YIOWHJzuzMkGzla*rWEO%KCitq zPNpp(PDA|s#>UMzWq4dqpF6&<@%Y)SD!%S*$8`QluANer^riNn?~z8Sh+vLAPi5|0 zJ3Y^Rr=nwfQ@D+aUR*|{$DCH9n$2mSGs~+B@2`Dtmhfao$<(8J9!Dg49gG&4+rt0( zZs`;aE8Yr|Gtq@iCv9;;**6){X6B zU1gvC!+iHTzbNaUtG+7agF<9;pkomzG!ElU0-Q)8LQ-ZmMAnEiSH>5YC#I*T7LidJ z*pg!ba%yeuv*tf!z|->mU)PE1=o=E3LYx{RA}UrWZ;A?5dK##zccI>Y#gPL?diTA5 zr}KKc=_5zQtoK?GV%!2xqV8v24==EM|4Hc9-ToSd zW4uOjkA;8FWBL9dP#6XM9%Ea89 z+I~YmV0gKHpL0ljL1toOj)!c5L(*%Rt$G_p9%&T5{F=Ffp|kPmru~2C=k`hDs4(wd zwNA=4F=a7B%aO&6!d2&G)$L1E`qXzWxN)s3){X5I#Y7=wk3dwr#asAgu z&HXuNH*E=%3=ZtMbzsUKMs=ayYxAmK%cdLte!KA6I)>*=bHX3gGV-xXtlbf_?wTsY zoq%gB3WC*jd@zak5 z?n+NRH7RPxjI##=KM7gv-S%P2Z6E!_o4Ok`_Aj`8%XiMz?{9?ot{j?{G-dA;iEnGz zCBIG+Vb)rgav~@)v-V8T6s|u>XP;gE_wLog`}f`-c&T|?tN&?aP_w1`w-wJ?{GYL# z2P_lOd2n*B@*n5@hQHz^Ed1>H4jM$cF@>+3@i=gi=IfM?MGw99u6*;dYR!e3ySL|L z-hHxsd-@(`P!jqVST*S^(3ke$aA0H-VL+|2VU@1}Pz@@;okAcLcz`#mX~;DTsKQ2o z03ef)X*lH()i=6@$R#_d!a{(9K+obd5L)8Eq8$|L=q4kVkf53Y0X6}{3%Q;^*9lSy z3sO)CiU3EL5jyqpX#&BbOtf(h~vVSdd&rtnv7Q4msa}3MK@| sV Date: Sun, 2 Jan 2022 23:57:00 +0530 Subject: [PATCH 3/4] Change licence year --- .../initializer/serviceCatalog/ServiceCatalogDeployer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java index b9733df90b..6be21ce116 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. * * WSO2 Inc. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except From 4a7d51d3c78325301409c59c055bc3f9e1d0eeec Mon Sep 17 00:00:00 2001 From: Arunan Date: Wed, 5 Jan 2022 23:28:56 +0530 Subject: [PATCH 4/4] Add review suggestions --- .../AppDeployerServiceComponent.java | 1 + .../application/deployer/CappDeployer.java | 17 ++++- .../ServiceCatalogDeployer.java | 9 +-- .../ServiceCatalogExecutor.java | 76 ------------------- .../initializer/utils/Constants.java | 1 + .../utils/ServiceCatalogUtils.java | 59 ++++++++------ 6 files changed, 56 insertions(+), 107 deletions(-) delete mode 100644 components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java index 623e118d77..9304fd3655 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java @@ -159,6 +159,7 @@ private void addCAppDeployer(DeploymentEngine deploymentEngine) { // Initialize CApp deployer here CappDeployer cappDeployer = new CappDeployer(); cappDeployer.setDirectory(artifactRepoPath + DeploymentConstants.CAPP_DIR_NAME); + cappDeployer.setSecretCallbackHandlerService(secretCallbackHandlerService); cappDeployer.init(configCtx); // Register application deployment handlers diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java index f95ce0b749..55feb01e1e 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java @@ -33,6 +33,7 @@ import org.apache.synapse.config.SynapseConfigUtils; import org.apache.synapse.config.SynapseConfiguration; import org.apache.synapse.api.API; +import org.wso2.carbon.securevault.SecretCallbackHandlerService; import org.wso2.micro.application.deployer.AppDeployerUtils; import org.wso2.micro.application.deployer.CarbonApplication; import org.wso2.micro.application.deployer.config.ApplicationConfiguration; @@ -101,6 +102,11 @@ public class CappDeployer extends AbstractDeployer { */ private Map serviceCatalogConfiguration; + /** + * SecretCallbackHandlerService to read Service Catalog Configs + */ + private SecretCallbackHandlerService secretCallbackHandlerService; + public void init(ConfigurationContext configurationContext) { if (log.isDebugEnabled()) { log.debug("Initializing Capp Deployer.."); @@ -113,8 +119,9 @@ public void init(ConfigurationContext configurationContext) { FileManipulator.deleteDir(appUnzipDir); if (ServiceCatalogUtils.isServiceCatalogEnabled()) { - serviceCatalogExecutor = Executors.newFixedThreadPool(10); - serviceCatalogConfiguration = ServiceCatalogUtils.readConfiguration(); + serviceCatalogConfiguration = ServiceCatalogUtils.readConfiguration(secretCallbackHandlerService); + serviceCatalogExecutor = Executors.newFixedThreadPool( + ServiceCatalogUtils.getExecutorThreadCount(serviceCatalogConfiguration, 10)); } } @@ -213,7 +220,7 @@ private void deployCarbonApps(String artifactPath) throws CarbonException { faultyCAppObjects.add(currentApp); faultyCapps.add(cAppName); } - if (ServiceCatalogUtils.isServiceCatalogEnabled()) { + if (serviceCatalogConfiguration != null && !faultyCapps.contains(cAppName)) { ServiceCatalogDeployer serviceDeployer = new ServiceCatalogDeployer(cAppName, ((CarbonAxisConfigurator) axisConfig.getAxisConfiguration().getConfigurator()).getRepoLocation(), serviceCatalogConfiguration); @@ -684,6 +691,10 @@ public void cleanup() { faultyCAppObjects.clear(); } + public void setSecretCallbackHandlerService(SecretCallbackHandlerService secretCallbackHandlerService) { + this.secretCallbackHandlerService = secretCallbackHandlerService; + } + /** * Partially building the API to get the API name * diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java index 6be21ce116..d0e71831f5 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java @@ -31,13 +31,12 @@ public class ServiceCatalogDeployer implements Runnable { - private String cAppName; - private Map serviceCatalogConfiguration; - private final String repoLocation; - + private static final Log log = LogFactory.getLog(ServiceCatalogDeployer.class); private static final String CAPP_UNZIP_DIR; - private static final Log log = LogFactory.getLog(ServiceCatalogDeployer.class); + private final String cAppName; + private final Map serviceCatalogConfiguration; + private final String repoLocation; public ServiceCatalogDeployer(String name, String repoLocation, Map serviceCatalogConfiguration) { this.cAppName = name; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java deleted file mode 100644 index 2cb72ea52b..0000000000 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.wso2.micro.integrator.initializer.serviceCatalog; - -import org.wso2.carbon.securevault.SecretCallbackHandlerService; - -import java.io.File; -import java.util.Map; - -import static org.wso2.micro.integrator.initializer.utils.Constants.*; -import static org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils.*; - -/** - * This class will publish the metadata to the service catalog in a separate thread. - */ -public class ServiceCatalogExecutor implements Runnable { - - - private static final String CAPP_UNZIP_DIR; - - private final String repoLocation; - private final SecretCallbackHandlerService secretCallbackHandlerService; - - public ServiceCatalogExecutor(String repoLocation, - SecretCallbackHandlerService secretCallbackHandlerService) { - this.repoLocation = repoLocation; - this.secretCallbackHandlerService = secretCallbackHandlerService; - } - - static { - String javaTmpDir = System.getProperty("java.io.tmpdir"); - CAPP_UNZIP_DIR = javaTmpDir.endsWith(File.separator) ? javaTmpDir + SERVICE_CATALOG : - javaTmpDir + File.separator + SERVICE_CATALOG; - } - - @Override - public void run() { - // check pre-conditions - if (!checkPreConditions()) return; - - Map apimConfigs = readConfiguration(secretCallbackHandlerService); - - // call service catalog and get all services - Map md5MapOfAllService = getAllServices(apimConfigs); - if (md5MapOfAllService == null) return; // error occurred while getting all services - - // create temporary directory to hold metadata. - if (!createTemporaryFolders(CAPP_UNZIP_DIR)) return; - - File tempDir = new File(CAPP_UNZIP_DIR, TEMP_FOLDER_NAME); - - // extract CAPPs and copy metadata to temp directory. - if (!extractMetadataFromCAPPs(tempDir, repoLocation, md5MapOfAllService)) return; - - // create the payload.zip file with extracted metadata - if (!archiveDir(CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME, tempDir.getPath())) return; - - // publish to service catalog endpoint. - publishToAPIM(apimConfigs, CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME); - } -} diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java index 0fdc005e66..cc87317772 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java @@ -41,6 +41,7 @@ public class Constants { public static final String ENABLE = "enable"; public static final String USER_NAME = "username"; public static final String PASSWORD = "password"; + public static final String SERVICE_CATALOG_EXECUTOR_THREADS = "executor_threads"; public static final String SERVICE_CATALOG_PUBLISH_ENDPOINT = "api/am/service-catalog/v0/services/import?overwrite" + "=true"; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java index 9aeecef234..295e8748ab 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java @@ -384,6 +384,7 @@ public static Map readConfiguration(SecretCallbackHandlerService String userName = catalogProperties.get(USER_NAME); String password = catalogProperties.get(PASSWORD); + String executorThreads = catalogProperties.get(SERVICE_CATALOG_EXECUTOR_THREADS); if (secretResolver == null) { secretResolver = SecretResolverFactory.create((OMElement) null, false); } @@ -402,29 +403,7 @@ public static Map readConfiguration(SecretCallbackHandlerService configMap.put(APIM_HOST, apimHost); configMap.put(USER_NAME, userName); configMap.put(PASSWORD, password); - return configMap; - } - - /** - * Read APIM host configurations from deployment.toml file. - * - * @param secretCallbackHandlerService secret callback handler reference. - * @return map of resolved values. - */ - public static Map readConfiguration() { - Map configMap = new HashMap<>(); - Map catalogProperties = - (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( - SERVICE_CATALOG_CONFIG)).get(0); - - String apimHost = catalogProperties.get(APIM_HOST); - - String userName = catalogProperties.get(USER_NAME); - String password = catalogProperties.get(PASSWORD); - - configMap.put(APIM_HOST, apimHost); - configMap.put(USER_NAME, userName); - configMap.put(PASSWORD, password); + configMap.put(SERVICE_CATALOG_EXECUTOR_THREADS, executorThreads); return configMap; } @@ -895,6 +874,11 @@ private static String getProxyServiceUrlFromMetadata(File yamlFile) { return currentServiceUrl; } + /** + * Checks whether Service Catalog configuration is Enabled. + * + * @return whether Service Catalog is enabled. + */ public static boolean isServiceCatalogEnabled() { Map catalogProperties; if (ConfigParser.getParsedConfigs().get(SERVICE_CATALOG_CONFIG) != null) { @@ -907,4 +891,33 @@ public static boolean isServiceCatalogEnabled() { } return false; } + + /** + * Retrieve Executor Thread configuration for Service Catalog. + * + * @param serviceCatalogConfig Map containing deployment.toml configuration + * @param def Default value + * + * @return Executor Thread count. + */ + public static int getExecutorThreadCount(Map serviceCatalogConfig, int def) { + String executorThreads = serviceCatalogConfig.get(SERVICE_CATALOG_EXECUTOR_THREADS); + int threads; + if (executorThreads != null) { + try { + threads = Integer.parseInt(executorThreads); + if (log.isDebugEnabled()) { + log.debug("Service Catalog Executor Thread count is set to " + threads); + } + } catch (NumberFormatException e) { + log.warn("Invalid Service Catalog Executor Thread count. Setting to default " + def); + return def; + } + return threads ; + } + if (log.isDebugEnabled()) { + log.debug("Service Catalog Executor Thread count is not defined. Setting to default " + def); + } + return def; + } }