Skip to content

Commit

Permalink
feat: query global config from arex-api (#209)
Browse files Browse the repository at this point in the history
  • Loading branch information
@coryhh
coryhh authored Nov 9, 2023
1 parent 924b8e6 commit 6972ecc
Show file tree
Hide file tree
Showing 11 changed files with 138 additions and 87 deletions.
4 changes: 2 additions & 2 deletions arex-schedule-web-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@
<parent>
<artifactId>arex-schedule-parent</artifactId>
<groupId>com.arextest</groupId>
<version>1.0.39</version>
<version>1.0.40</version>
</parent>

<profiles>
Expand Down Expand Up @@ -280,5 +280,5 @@
</properties>
</profile>
</profiles>
<version>1.0.39</version>
<version>1.0.40</version>
</project>
175 changes: 114 additions & 61 deletions arex-schedule-web-api/src/main/java/com/arextest/schedule/comparer/CompareConfigService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.arextest.schedule.model.converter.ReplayConfigConverter;
import com.arextest.schedule.progress.ProgressEvent;
import com.arextest.schedule.utils.MapUtils;
import com.arextest.web.model.contract.contracts.config.SystemConfig;
import com.arextest.web.model.contract.contracts.config.replay.ReplayCompareConfig;
import com.arextest.web.model.contract.contracts.config.replay.ReplayCompareConfig.DependencyComparisonItem;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -29,6 +30,7 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.ResponseEntity;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.stereotype.Component;

/**
Expand All @@ -46,53 +48,60 @@ public final class CompareConfigService {
private CacheProvider redisCacheProvider;
@Value("${arex.api.config.comparison.summary.url}")
private String summaryConfigUrl;
@Value("${arex.api.config.system.url}")
private String systemConfigUrl;
@Resource
private ProgressEvent progressEvent;
@Resource
private ObjectMapper objectMapper;

private static Map<String, ComparisonInterfaceConfig> convertOperationConfig(
List<ReplayCompareConfig.ReplayComparisonItem> operationConfigs) {
Map<String, ComparisonInterfaceConfig> res = new HashMap<>();
private static final RetryTemplate RETRY_TEMPLATE = RetryTemplate.builder().maxAttempts(3)
.fixedBackoff(200L)
.build();
private static final long DEFAULT_COMPARE_IGNORE_TIME_PRECISION_MILLIS = 2000;
private static final boolean DEFAULT_COMPARE_NAME_TO_LOWER = true;
private static final boolean DEFAULT_COMPARE_NULL_EQUALS_EMPTY = true;
private static final boolean DEFAULT_COMPARE_ONLY_COMPARE_COINCIDENT_COLUMN = true;
private static final boolean DEFAULT_COMPARE_SELECT_IGNORE_COMPARE = true;
private static final boolean DEFAULT_COMPARE_UUID_IGNORE = true;

for (ReplayCompareConfig.ReplayComparisonItem source : operationConfigs) {
String operationId = source.getOperationId();
if (StringUtils.isBlank(operationId)) {
LOGGER.warn("operation id is blank, operationId: {}", operationId);
continue;
}
private static SystemConfig systemConfig = null;

ComparisonInterfaceConfig converted = ReplayConfigConverter.INSTANCE.interfaceDaoFromDto(
source);
List<ReplayCompareConfig.DependencyComparisonItem> sourceDependencyConfigs = source.getDependencyComparisonItems();
converted.setDependencyConfigMap(convertDependencyConfig(sourceDependencyConfigs));
DependencyComparisonItem defaultDependencyComparisonItem = source.getDefaultDependencyComparisonItem();
converted.setDefaultDependencyConfig(
ReplayConfigConverter.INSTANCE.dependencyDaoFromDto(defaultDependencyComparisonItem));
res.put(operationId, converted);
public SystemConfig getComparisonSystemConfig() {

if (systemConfig != null) {
return systemConfig;
}

return res;
}
ResponseEntity<GenericResponseType<SystemConfig>> response = RETRY_TEMPLATE.execute(context -> {
ResponseEntity<GenericResponseType<SystemConfig>> temp = httpWepServiceApiClient.get(
systemConfigUrl, Collections.emptyMap(),
new ParameterizedTypeReference<GenericResponseType<SystemConfig>>() {
});

private static Map<String, ComparisonDependencyConfig> convertDependencyConfig(
List<ReplayCompareConfig.DependencyComparisonItem> dependencyConfigs) {
Map<String, ComparisonDependencyConfig> res = new HashMap<>();

for (ReplayCompareConfig.DependencyComparisonItem source : dependencyConfigs) {
if (CollectionUtils.isEmpty(source.getOperationTypes()) || StringUtils.isBlank(
source.getOperationName())) {
LOGGER.warn("dependency type or name is blank, dependencyId: {}", source.getDependencyId());
continue;
if (temp == null || temp.getBody() == null || temp.getBody().getBody() == null) {
throw new RuntimeException("get compare system config failed");
} else {
return temp;
}

String dependencyKey = ComparisonDependencyConfig.dependencyKey(source);
ComparisonDependencyConfig converted = ReplayConfigConverter.INSTANCE.dependencyDaoFromDto(
source);
res.put(dependencyKey, converted);
}, retryContext -> null);

if (response == null || response.getBody() == null || response.getBody().getBody() == null) {
LOGGER.error("get compare system config failed");
SystemConfig defaultConfig = new SystemConfig();
defaultConfig.setCompareIgnoreTimePrecisionMillis(
DEFAULT_COMPARE_IGNORE_TIME_PRECISION_MILLIS);
defaultConfig.setCompareNameToLower(DEFAULT_COMPARE_NAME_TO_LOWER);
defaultConfig.setCompareNullEqualsEmpty(DEFAULT_COMPARE_NULL_EQUALS_EMPTY);
defaultConfig.setOnlyCompareCoincidentColumn(DEFAULT_COMPARE_ONLY_COMPARE_COINCIDENT_COLUMN);
defaultConfig.setSelectIgnoreCompare(DEFAULT_COMPARE_SELECT_IGNORE_COMPARE);
defaultConfig.setUuidIgnore(DEFAULT_COMPARE_UUID_IGNORE);
systemConfig = defaultConfig;
} else {
systemConfig = response.getBody().getBody();
}

return res;
return systemConfig;
}

public void preload(ReplayPlan plan) {
Expand Down Expand Up @@ -125,33 +134,6 @@ public void preload(ReplayPlan plan) {
progressEvent.onCompareConfigLoaded(plan);
}

private Map<String, ComparisonInterfaceConfig> getReplayComparisonConfig(
ReplayPlan plan) {
Map<String, String> urlVariables = Collections.singletonMap("appId", plan.getAppId());

ResponseEntity<GenericResponseType<ReplayCompareConfig>> replayComparisonConfigEntity =
httpWepServiceApiClient.get(summaryConfigUrl, urlVariables,
new ParameterizedTypeReference<GenericResponseType<ReplayCompareConfig>>() {
});

if (replayComparisonConfigEntity == null) {
return new HashMap<>();
}

List<ReplayCompareConfig.ReplayComparisonItem> operationConfigs = Optional.ofNullable(
replayComparisonConfigEntity.getBody())
.map(GenericResponseType::getBody)
.map(ReplayCompareConfig::getReplayComparisonItems)
.orElse(Collections.emptyList());
// TODO: add the log of the ComparisonConfig

// converts
Map<String, ComparisonInterfaceConfig> opConverted = convertOperationConfig(operationConfigs);
this.setContractChangeFlag(opConverted,
replayComparisonConfigEntity.getBody().getBody().getSkipAssemble());
return opConverted;
}

public ComparisonInterfaceConfig loadInterfaceConfig(ReplayActionItem actionItem) {
return this.loadInterfaceConfig(actionItem.getId());
}
Expand Down Expand Up @@ -193,6 +175,77 @@ public ComparisonGlobalConfig loadGlobalConfig(String planId) {
return ComparisonGlobalConfig.empty();
}

private Map<String, ComparisonInterfaceConfig> getReplayComparisonConfig(
ReplayPlan plan) {
Map<String, String> urlVariables = Collections.singletonMap("appId", plan.getAppId());

ResponseEntity<GenericResponseType<ReplayCompareConfig>> replayComparisonConfigEntity =
httpWepServiceApiClient.get(summaryConfigUrl, urlVariables,
new ParameterizedTypeReference<GenericResponseType<ReplayCompareConfig>>() {
});

if (replayComparisonConfigEntity == null) {
return new HashMap<>();
}

List<ReplayCompareConfig.ReplayComparisonItem> operationConfigs = Optional.ofNullable(
replayComparisonConfigEntity.getBody())
.map(GenericResponseType::getBody)
.map(ReplayCompareConfig::getReplayComparisonItems)
.orElse(Collections.emptyList());
// TODO: add the log of the ComparisonConfig

// converts
Map<String, ComparisonInterfaceConfig> opConverted = convertOperationConfig(operationConfigs);
this.setContractChangeFlag(opConverted,
replayComparisonConfigEntity.getBody().getBody().getSkipAssemble());
return opConverted;
}

private static Map<String, ComparisonInterfaceConfig> convertOperationConfig(
List<ReplayCompareConfig.ReplayComparisonItem> operationConfigs) {
Map<String, ComparisonInterfaceConfig> res = new HashMap<>();

for (ReplayCompareConfig.ReplayComparisonItem source : operationConfigs) {
String operationId = source.getOperationId();
if (StringUtils.isBlank(operationId)) {
LOGGER.warn("operation id is blank, operationId: {}", operationId);
continue;
}

ComparisonInterfaceConfig converted = ReplayConfigConverter.INSTANCE.interfaceDaoFromDto(
source);
List<ReplayCompareConfig.DependencyComparisonItem> sourceDependencyConfigs = source.getDependencyComparisonItems();
converted.setDependencyConfigMap(convertDependencyConfig(sourceDependencyConfigs));
DependencyComparisonItem defaultDependencyComparisonItem = source.getDefaultDependencyComparisonItem();
converted.setDefaultDependencyConfig(
ReplayConfigConverter.INSTANCE.dependencyDaoFromDto(defaultDependencyComparisonItem));
res.put(operationId, converted);
}

return res;
}

private static Map<String, ComparisonDependencyConfig> convertDependencyConfig(
List<ReplayCompareConfig.DependencyComparisonItem> dependencyConfigs) {
Map<String, ComparisonDependencyConfig> res = new HashMap<>();

for (ReplayCompareConfig.DependencyComparisonItem source : dependencyConfigs) {
if (CollectionUtils.isEmpty(source.getOperationTypes()) || StringUtils.isBlank(
source.getOperationName())) {
LOGGER.warn("dependency type or name is blank, dependencyId: {}", source.getDependencyId());
continue;
}

String dependencyKey = ComparisonDependencyConfig.dependencyKey(source);
ComparisonDependencyConfig converted = ReplayConfigConverter.INSTANCE.dependencyDaoFromDto(
source);
res.put(dependencyKey, converted);
}

return res;
}

private <T> T byteToObject(byte[] bytes, Class<T> tClass) {
try {
return objectMapper.readValue(bytes, tClass);
Expand Down
4 changes: 0 additions & 4 deletions ...java/com/arextest/schedule/comparer/impl/DefaultCustomComparisonConfigurationHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,6 @@ public ReplayComparisonConfig pickConfig(ComparisonGlobalConfig globalConfig,
public CompareOptions buildSkdOption(String category, ReplayComparisonConfig compareConfig) {
CompareOptions options = new CompareOptions();
options.putCategoryType(category);
// todo: the switch of "sqlBodyParse" and "onlyCompareCoincidentColumn"
// need get from ReplayComparisonConfig
options.putSelectIgnoreCompare(true);
options.putOnlyCompareCoincidentColumn(true);
if (Objects.equals(category, MockCategoryType.DATABASE.getName())) {
options.putExclusions(DEFAULT_DATABASE_IGNORE);
}
Expand Down
29 changes: 13 additions & 16 deletions ...eb-api/src/main/java/com/arextest/schedule/comparer/impl/DefaultReplayResultComparer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,16 @@
import com.arextest.schedule.model.config.ReplayComparisonConfig;
import com.arextest.schedule.progress.ProgressTracer;
import com.arextest.schedule.service.MetricService;
import java.io.IOException;
import java.io.InputStream;
import com.arextest.web.model.contract.contracts.config.SystemConfig;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -54,19 +53,17 @@ public class DefaultReplayResultComparer implements ReplayResultComparer {
private static final String IGNORE_TIME_PRECISION_MILLIS_KEY = "ignore.time.precision.millis";
private static long ignoreTimePrecisionMillis = 2000;

static {
Properties props = new Properties();
try (InputStream input = DefaultReplayResultComparer.class.getResourceAsStream(
APPLICATION_PROPERTIES_NAME)) {
props.load(input);
ignoreTimePrecisionMillis = Long.parseLong(
props.getProperty(IGNORE_TIME_PRECISION_MILLIS_KEY));
} catch (IOException e) {
LOGGER.error("failed to get ignore time precision millis {}", e.getMessage(), e);
}

COMPARE_INSTANCE.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true)
.putIgnoredTimePrecision(ignoreTimePrecisionMillis);
@PostConstruct
public void init() {
SystemConfig comparisonSystemConfig = compareConfigService.getComparisonSystemConfig();
COMPARE_INSTANCE.getGlobalOptions()
.putNameToLower(comparisonSystemConfig.getCompareNameToLower())
.putNullEqualsEmpty(comparisonSystemConfig.getCompareNullEqualsEmpty())
.putIgnoredTimePrecision(comparisonSystemConfig.getCompareIgnoreTimePrecisionMillis())
.putIgnoreNodeSet(comparisonSystemConfig.getIgnoreNodeSet())
.putSelectIgnoreCompare(comparisonSystemConfig.getSelectIgnoreCompare())
.putOnlyCompareCoincidentColumn(comparisonSystemConfig.getOnlyCompareCoincidentColumn())
.putUuidIgnore(comparisonSystemConfig.getUuidIgnore());
}

private final CompareConfigService compareConfigService;
Expand Down
1 change: 1 addition & 0 deletions ...web-api/src/main/java/com/arextest/schedule/model/config/AbstractCompareDetailConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
/**
* Created by rchen9 on 2022/9/19.
*/
@Deprecated
@Getter
@Setter
public abstract class AbstractCompareDetailConfig {
Expand Down
1 change: 1 addition & 0 deletions ...ule-web-api/src/main/java/com/arextest/schedule/model/config/CompareExclusionsConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
/**
* Created by rchen9 on 2022/9/19.
*/
@Deprecated
@Data
public class CompareExclusionsConfig extends AbstractCompareDetailConfig {

Expand Down
1 change: 1 addition & 0 deletions ...ule-web-api/src/main/java/com/arextest/schedule/model/config/CompareInclusionsConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
/**
* Created by rchen9 on 2022/9/19.
*/
@Deprecated
@Data
public class CompareInclusionsConfig extends AbstractCompareDetailConfig {

Expand Down
1 change: 1 addition & 0 deletions ...edule-web-api/src/main/java/com/arextest/schedule/model/config/CompareListSortConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
/**
* Created by rchen9 on 2022/9/20.
*/
@Deprecated
@Data
public class CompareListSortConfig extends AbstractCompareDetailConfig {

Expand Down
1 change: 1 addition & 0 deletions ...dule-web-api/src/main/java/com/arextest/schedule/model/config/CompareReferenceConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
/**
* Created by rchen9 on 2022/9/20.
*/
@Deprecated
@Data
public class CompareReferenceConfig extends AbstractCompareDetailConfig {

Expand Down
2 changes: 1 addition & 1 deletion arex-schedule-web-api/src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ arex.storage.replayCase.url=${arex.storage.service.api}/api/storage/replay/query
arex.storage.postProcess.url=${arex.storage.service.api}/api/storage/replay/postprocessResult
#for arex-api
arex.api.init.url=${arex.api.service.api}/api/report/init
# deprecated arex.api.push.compareResult.url=${arex.api.service.api}/api/report/pushCompareResults
arex.api.push.compareResult.url=${arex.api.service.api}/api/report/analyzeCompareResults
arex.api.push.replayStatus.url=${arex.api.service.api}/api/report/pushReplayStatus
arex.api.update.report.info.url=${arex.api.service.api}/api/report/updateReportInfo
arex.api.remove.records.url=${arex.api.service.api}/api/report/removeRecordsAndScenes
#for schedule
arex.api.config.system.url=${arex.api.service.api}/api/system/config/list
arex.api.config.comparison.summary.url=${arex.api.service.api}/api/config/comparison/summary/queryByAppId/{appId}
arex.api.config.schedule.url=${arex.api.service.api}/api/config/schedule/useResult/appId/{appId}
arex.api.config.applicationService.url=${arex.api.service.api}/api/config/applicationService/useResultAsList/appId/{appId}
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@
<packaging>pom</packaging>
<properties>
<commons-lang3.version>3.3.2</commons-lang3.version>
<compare-sdk.version>0.1.27</compare-sdk.version>
<compare-sdk.version>0.2.0</compare-sdk.version>
<fastjson2.version>2.0.23</fastjson2.version>
<guava.version>20.0</guava.version>
<jacoco-maven-plugin.version>0.7.5.201505241946</jacoco-maven-plugin.version>
Expand Down Expand Up @@ -294,7 +294,7 @@
<sonar.jacoco.reportPath>${jacoco.report.path}</sonar.jacoco.reportPath>
<!-- Users could specify this property as true to skip the deploy process -->
<spring.boot.version>2.6.3</spring.boot.version>
<web-contract.version>0.6.0.4</web-contract.version>
<web-contract.version>0.6.0.7</web-contract.version>
</properties>

<scm>
Expand All @@ -304,5 +304,5 @@
<url>https://github.com/arextest/arex-replay-schedule</url>
</scm>
<url>https://github.com/arextest/arex-replay-schedule</url>
<version>1.0.39</version>
<version>1.0.40</version>
</project>

0 comments on commit 6972ecc

Please sign in to comment.