From 25d2fc6f76ea846d2f28ef66074c84c74f320f2f Mon Sep 17 00:00:00 2001 From: rchen9 Date: Thu, 9 Nov 2023 10:24:44 +0800 Subject: [PATCH 1/2] feat: exclude noise --- .../schedule/beans/ControllerException.java | 33 +++ .../dao/mongodb/ReplayNoiseRepository.java | 46 ++++ .../model/converter/ReplayNoiseConverter.java | 7 +- .../dao/mongodb/ReplayNoiseCollection.java | 2 + .../ExcludeNoiseRequestType.java | 50 +++++ .../QueryNoiseResponseType.java | 9 +- .../noiseidentify/ReplayNoiseItemDto.java | 2 + .../noiseidentify/ReplayNoiseStatus.java | 9 + .../model/noiseidentify/UpdateNoiseItem.java | 13 ++ .../noise/ReplayNoiseHandlerService.java | 197 ++++++++++++++++++ .../service/noise/ReplayNoiseIdentify.java | 2 - .../noise/ReplayNoiseIdentifyService.java | 81 ------- .../arextest/schedule/utils/ListUtils.java | 14 ++ .../web/controller/ReplayNoiseController.java | 24 ++- .../src/main/resources/application.properties | 1 + .../src/main/resources/log4j2-dev.xml | 37 ++++ 16 files changed, 434 insertions(+), 93 deletions(-) create mode 100644 arex-schedule-web-api/src/main/java/com/arextest/schedule/beans/ControllerException.java create mode 100644 arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ExcludeNoiseRequestType.java rename arex-schedule-web-api/src/main/java/com/arextest/schedule/model/{report => noiseidentify}/QueryNoiseResponseType.java (86%) create mode 100644 arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseStatus.java create mode 100644 arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/UpdateNoiseItem.java create mode 100644 arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseHandlerService.java create mode 100644 arex-schedule-web-api/src/main/resources/log4j2-dev.xml diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/beans/ControllerException.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/beans/ControllerException.java new file mode 100644 index 00000000..2b7cc9aa --- /dev/null +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/beans/ControllerException.java @@ -0,0 +1,33 @@ +package com.arextest.schedule.beans; + +import com.arextest.schedule.model.CommonResponse; +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.ObjectError; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@Slf4j +@RestControllerAdvice +public class ControllerException { + + @ExceptionHandler(MethodArgumentNotValidException.class) + public ResponseEntity handleValidException(MethodArgumentNotValidException e) { + // setting application/json;charset=UTF-8 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON_UTF8); + + List allErrors = e.getBindingResult().getAllErrors(); + String message = allErrors.stream().map(s -> s.getDefaultMessage()) + .collect(Collectors.joining(";")); + + return new ResponseEntity<>(CommonResponse.badResponse(message), headers, + HttpStatus.BAD_REQUEST); + } +} diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/dao/mongodb/ReplayNoiseRepository.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/dao/mongodb/ReplayNoiseRepository.java index 61cbd76f..ce30bcff 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/dao/mongodb/ReplayNoiseRepository.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/dao/mongodb/ReplayNoiseRepository.java @@ -3,8 +3,11 @@ import com.arextest.schedule.dao.mongodb.util.MongoHelper; import com.arextest.schedule.model.converter.ReplayNoiseConverter; import com.arextest.schedule.model.dao.mongodb.ReplayNoiseCollection; +import com.arextest.schedule.model.dao.mongodb.ReplayNoiseCollection.ReplayNoiseItemDao; import com.arextest.schedule.model.noiseidentify.ReplayNoiseDto; +import com.arextest.schedule.model.noiseidentify.UpdateNoiseItem; import com.arextest.schedule.utils.MapUtils; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -101,6 +104,44 @@ public List queryReplayNoise(@NotBlank String planId, String pla .collect(Collectors.toList()); } + public boolean updateReplayNoiseStatus(List updateNoiseItems) { + + try { + BulkOperations bulkOperations = + mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, ReplayNoiseCollection.class); + for (UpdateNoiseItem updateNoiseItem : updateNoiseItems) { + Map queryFields = updateNoiseItem.getQueryFields(); + Map updateFields = updateNoiseItem.getUpdateFields(); + if (MapUtils.isEmpty(queryFields) || MapUtils.isEmpty(updateFields)) { + continue; + } + + List criteriaList = new ArrayList<>(); + for (Map.Entry entry : queryFields.entrySet()) { + if (entry.getKey() == null) { + continue; + } + criteriaList.add(Criteria.where(entry.getKey()).is(entry.getValue())); + } + if (CollectionUtils.isEmpty(criteriaList)) { + continue; + } + Query query = new Query().addCriteria(new Criteria().andOperator(criteriaList)); + + Update update = MongoHelper.getUpdate(); + for (Map.Entry entry : updateFields.entrySet()) { + update.set(entry.getKey(), entry.getValue()); + } + bulkOperations.updateMulti(query, update); + } + bulkOperations.execute(); + } catch (RuntimeException e) { + LOGGER.error("ReplayNoiseRepository.updateReplayNoiseStatus error", e); + return false; + } + return true; + } + private void appendUpdate(Update update, Map needUpdateContent, String updateKey) { @@ -120,6 +161,11 @@ private void appendUpdate(Update update, MongoHelper.appendDot(updateKey, path, ReplayNoiseCollection.ReplayNoiseItemDao.FIELD_LOG_INDEXES), replayNoiseItemDao.getLogIndexes()); + if (replayNoiseItemDao.getStatus() != null) { + update.set( + MongoHelper.appendDot(updateKey, path, ReplayNoiseItemDao.FIELD_STATUS), + replayNoiseItemDao.getStatus()); + } update.inc( MongoHelper.appendDot(updateKey, path, ReplayNoiseCollection.ReplayNoiseItemDao.FIELD_PATH_COUNT), diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/converter/ReplayNoiseConverter.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/converter/ReplayNoiseConverter.java index 8f77e5db..f985749b 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/converter/ReplayNoiseConverter.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/converter/ReplayNoiseConverter.java @@ -1,10 +1,12 @@ package com.arextest.schedule.model.converter; import com.arextest.schedule.model.dao.mongodb.ReplayNoiseCollection; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType; import com.arextest.schedule.model.noiseidentify.ReplayNoiseDto; import com.arextest.schedule.model.noiseidentify.ReplayNoiseItemDto; import org.mapstruct.Mapper; import org.mapstruct.Mapping; +import org.mapstruct.Mappings; import org.mapstruct.factory.Mappers; /** @@ -25,5 +27,8 @@ public interface ReplayNoiseConverter { ReplayNoiseItemDto dtoFromDao(ReplayNoiseCollection.ReplayNoiseItemDao dao); -// ReplayNoiseDto.ReplayNoiseItemDto toReplayNoiseItemDto(ReplayNoiseItemDto dao); + @Mappings( + @Mapping(target = "nodeEntity", source = "nodePath") + ) + QueryNoiseResponseType.NoiseItem toNoiseItem(ReplayNoiseItemDto dto); } diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/dao/mongodb/ReplayNoiseCollection.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/dao/mongodb/ReplayNoiseCollection.java index 1aa5d189..a2197fbd 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/dao/mongodb/ReplayNoiseCollection.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/dao/mongodb/ReplayNoiseCollection.java @@ -46,5 +46,7 @@ public static class ReplayNoiseItemDao { private Integer caseCount; + private Integer status; + } } diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ExcludeNoiseRequestType.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ExcludeNoiseRequestType.java new file mode 100644 index 00000000..db35feba --- /dev/null +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ExcludeNoiseRequestType.java @@ -0,0 +1,50 @@ +package com.arextest.schedule.model.noiseidentify; + +import com.arextest.diff.model.log.NodeEntity; +import com.arextest.model.mock.MockCategoryType; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType.InterfaceNoiseItem; +import java.util.List; +import javax.validation.constraints.NotBlank; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +public class ExcludeNoiseRequestType { + + @NotBlank(message = "appId cannot be blank") + private String appId; + @NotBlank(message = "planId cannot be blank") + private String planId; + + private List interfaceNoiseItemList; + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class InterfaceExcludeNoiseItem { + private String operationId; + private List randomNoise; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class MockerExcludeNoiseItem { + + private MockCategoryType mockCategoryType; + private String operationName; + private String operationType; + private List noiseItemList; + } + + @Data + public static class ExcludeNoiseItem { + + private String identifier; + + private List nodeEntity; + + } + +} diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/report/QueryNoiseResponseType.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/QueryNoiseResponseType.java similarity index 86% rename from arex-schedule-web-api/src/main/java/com/arextest/schedule/model/report/QueryNoiseResponseType.java rename to arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/QueryNoiseResponseType.java index 5ff62446..58a1fa81 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/report/QueryNoiseResponseType.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/QueryNoiseResponseType.java @@ -1,4 +1,4 @@ -package com.arextest.schedule.model.report; +package com.arextest.schedule.model.noiseidentify; import com.arextest.diff.model.log.NodeEntity; import com.arextest.model.mock.MockCategoryType; @@ -39,12 +39,19 @@ public static class MockerNoiseItem { @Data public static class NoiseItem { + private String identifier; + private List nodeEntity; private List logIndexes; private String compareResultId; + /** + * @see ReplayNoiseStatus + */ + private Integer status; + } } diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseItemDto.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseItemDto.java index 32eb5b6f..dc850d12 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseItemDto.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseItemDto.java @@ -31,4 +31,6 @@ public class ReplayNoiseItemDto { public int getPathCount() { return subPaths.size(); } + + private Integer status; } \ No newline at end of file diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseStatus.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseStatus.java new file mode 100644 index 00000000..1c91f7d3 --- /dev/null +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/ReplayNoiseStatus.java @@ -0,0 +1,9 @@ +package com.arextest.schedule.model.noiseidentify; + +public interface ReplayNoiseStatus { + + int STATUS_NORMAL = 0; + + int STATUS_EXCLUDE = 1; + +} diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/UpdateNoiseItem.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/UpdateNoiseItem.java new file mode 100644 index 00000000..26f6322e --- /dev/null +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/model/noiseidentify/UpdateNoiseItem.java @@ -0,0 +1,13 @@ +package com.arextest.schedule.model.noiseidentify; + +import java.util.Map; +import lombok.Data; + +@Data +public class UpdateNoiseItem { + + private Map queryFields; + + private Map updateFields; + +} diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseHandlerService.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseHandlerService.java new file mode 100644 index 00000000..e3b0e976 --- /dev/null +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseHandlerService.java @@ -0,0 +1,197 @@ +package com.arextest.schedule.service.noise; + +import com.arextest.common.model.response.GenericResponseType; +import com.arextest.model.mock.MockCategoryType; +import com.arextest.schedule.client.HttpWepServiceApiClient; +import com.arextest.schedule.dao.mongodb.ReplayNoiseRepository; +import com.arextest.schedule.dao.mongodb.util.MongoHelper; +import com.arextest.schedule.model.converter.ReplayNoiseConverter; +import com.arextest.schedule.model.dao.mongodb.ReplayNoiseCollection; +import com.arextest.schedule.model.dao.mongodb.ReplayNoiseCollection.ReplayNoiseItemDao; +import com.arextest.schedule.model.noiseidentify.ExcludeNoiseRequestType; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType.InterfaceNoiseItem; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType.NoiseItem; +import com.arextest.schedule.model.noiseidentify.ReplayNoiseDto; +import com.arextest.schedule.model.noiseidentify.ReplayNoiseItemDto; +import com.arextest.schedule.model.noiseidentify.ReplayNoiseStatus; +import com.arextest.schedule.model.noiseidentify.UpdateNoiseItem; +import com.arextest.schedule.utils.ListUtils; +import com.arextest.schedule.utils.MapUtils; +import com.arextest.web.model.contract.contracts.config.replay.ComparisonExclusionsConfiguration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class ReplayNoiseHandlerService { + + @Resource + private ReplayNoiseRepository replayNoiseRepository; + + @Resource + private HttpWepServiceApiClient httpWepServiceApiClient; + + @Value("${arex.api.config.addExclusion.url}") + private String addExclusionConfigUrl; + + public QueryNoiseResponseType queryNoise(String planId, String planItemId) { + + List appNoiseList = new ArrayList<>(); + + List replayNoiseDtoList = replayNoiseRepository.queryReplayNoise(planId, + planItemId); + Map> interfaceReplayNoiseMap = + replayNoiseDtoList.stream().collect(Collectors.groupingBy(ReplayNoiseDto::getOperationId)); + for (Map.Entry> interfaceReplayNoiseEntry : interfaceReplayNoiseMap.entrySet()) { + String operationId = interfaceReplayNoiseEntry.getKey(); + List interfaceReplayNoise = interfaceReplayNoiseEntry.getValue(); + if (CollectionUtils.isEmpty(interfaceReplayNoise)) { + continue; + } + + List randomNoiseList = new ArrayList<>(); + List disorderedArrayNoise = new ArrayList<>(); + + for (ReplayNoiseDto replayNoiseDto : interfaceReplayNoise) { + String categoryName = replayNoiseDto.getCategoryName(); + MockCategoryType mockCategoryType = MockCategoryType.create(categoryName); + + Map mayIgnoreItems = replayNoiseDto.getMayIgnoreItems(); + if (MapUtils.isNotEmpty(mayIgnoreItems)) { + Set> entries = mayIgnoreItems.entrySet(); + + List randomNoiseItems = new ArrayList<>(entries.size()); + for (Entry itemDtoEntry : entries) { + String identifier = itemDtoEntry.getKey(); + ReplayNoiseItemDto value = itemDtoEntry.getValue(); + NoiseItem noiseItem = ReplayNoiseConverter.INSTANCE.toNoiseItem(value); + noiseItem.setIdentifier(identifier); + randomNoiseItems.add(noiseItem); + } + if (CollectionUtils.isNotEmpty(randomNoiseItems)) { + randomNoiseList.add(new QueryNoiseResponseType.MockerNoiseItem(mockCategoryType, + replayNoiseDto.getOperationName(), categoryName, randomNoiseItems)); + } + } + + Map mayDisorderItems = replayNoiseDto.getMayDisorderItems(); + if (MapUtils.isNotEmpty(mayDisorderItems)) { + Set> entries = mayDisorderItems.entrySet(); + List disorderedArrayNoiseItems = new ArrayList<>( + entries.size()); + for (Entry itemDtoEntry : entries) { + + String identifier = itemDtoEntry.getKey(); + ReplayNoiseItemDto value = itemDtoEntry.getValue(); + // XXX: simple array filter, improve: more accurate recommendations + if (value.getPathCount() < 2) { + continue; + } + NoiseItem noiseItem = ReplayNoiseConverter.INSTANCE.toNoiseItem(value); + noiseItem.setIdentifier(identifier); + } + if (CollectionUtils.isNotEmpty(disorderedArrayNoiseItems)) { + disorderedArrayNoise.add(new QueryNoiseResponseType.MockerNoiseItem(mockCategoryType, + replayNoiseDto.getOperationName(), categoryName, disorderedArrayNoiseItems)); + } + } + } + + if (CollectionUtils.isNotEmpty(randomNoiseList) || CollectionUtils.isNotEmpty( + disorderedArrayNoise)) { + appNoiseList.add( + new QueryNoiseResponseType.InterfaceNoiseItem(operationId, randomNoiseList, + disorderedArrayNoise)); + } + } + QueryNoiseResponseType result = new QueryNoiseResponseType(); + result.setInterfaceNoiseItemList(appNoiseList); + return result; + } + + + public boolean excludeNoise(ExcludeNoiseRequestType excludeNoiseRequestType) { + + List updateNoiseItemList = new ArrayList<>(); + List exclusionConfigs = new ArrayList<>(); + + List interfaceNoiseItemList = Optional.ofNullable( + excludeNoiseRequestType.getInterfaceNoiseItemList()).orElse( + Collections.emptyList()); + for (InterfaceNoiseItem interfaceNoiseItem : interfaceNoiseItemList) { + String operationId = interfaceNoiseItem.getOperationId(); + List randomNoiseList = Optional.ofNullable( + interfaceNoiseItem.getRandomNoise()).orElse(Collections.emptyList()); + for (QueryNoiseResponseType.MockerNoiseItem mockerNoiseItem : randomNoiseList) { + UpdateNoiseItem updateNoiseItem = new UpdateNoiseItem(); + updateNoiseItem.setQueryFields( + MapUtils.createMap( + ReplayNoiseCollection.FIELD_PLAN_ID, + excludeNoiseRequestType.getPlanId(), + ReplayNoiseCollection.FIELD_OPERATION_ID, + operationId, + ReplayNoiseCollection.FIELD_CATEGORY_NAME, + mockerNoiseItem.getOperationType() + ) + ); + + Map updateFields = new HashMap<>(); + List noiseItemList = Optional.ofNullable( + mockerNoiseItem.getNoiseItemList()).orElse(Collections.emptyList()); + for (QueryNoiseResponseType.NoiseItem noiseItem : noiseItemList) { + String identifier = noiseItem.getIdentifier(); + // add exclude noise item + updateFields.put(MongoHelper.appendDot(ReplayNoiseCollection.FIELD_MAY_IGNORE_ITEMS, + identifier, ReplayNoiseItemDao.FIELD_STATUS), ReplayNoiseStatus.STATUS_EXCLUDE); + + // add exclusions to config + ComparisonExclusionsConfiguration exclusionConfig = new ComparisonExclusionsConfiguration(); + exclusionConfig.setAppId(excludeNoiseRequestType.getAppId()); + exclusionConfig.setOperationId(operationId); + if (!mockerNoiseItem.getMockCategoryType().isEntryPoint()) { + exclusionConfig.setOperationType(mockerNoiseItem.getOperationType()); + exclusionConfig.setOperationName(mockerNoiseItem.getOperationName()); + } + List excludePath = ListUtils.getFuzzyPathStrList(noiseItem.getNodeEntity()); + if (excludePath != null) { + exclusionConfig.setExclusions(excludePath); + exclusionConfigs.add(exclusionConfig); + } + } + updateNoiseItem.setUpdateFields(updateFields); + updateNoiseItemList.add(updateNoiseItem); + } + } + + boolean result = replayNoiseRepository.updateReplayNoiseStatus(updateNoiseItemList); + if (!result) { + return false; + } + + if (CollectionUtils.isNotEmpty(exclusionConfigs)) { + GenericResponseType response = httpWepServiceApiClient.jsonPost(addExclusionConfigUrl, + exclusionConfigs, + GenericResponseType.class); + if (response == null || Objects.equals(response.getBody(), false)) { + LOGGER.error("add exclusion config failed, response: {}", response); + return false; + } + } + return true; + } + +} diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentify.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentify.java index 733b7fa6..b298a190 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentify.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentify.java @@ -3,7 +3,6 @@ import com.arextest.schedule.model.PlanExecutionContext; import com.arextest.schedule.model.ReplayActionCaseItem; import com.arextest.schedule.model.ReplayActionItem; -import com.arextest.schedule.model.report.QueryNoiseResponseType; import java.util.List; /** @@ -16,5 +15,4 @@ void noiseIdentify(List allCasesOfContext, void rerunNoiseAnalysisRecovery(List actionItems); - QueryNoiseResponseType queryNoise(String planId, String planItemId); } diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentifyService.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentifyService.java index 02157947..829fe7d4 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentifyService.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/ReplayNoiseIdentifyService.java @@ -1,6 +1,5 @@ package com.arextest.schedule.service.noise; -import com.arextest.model.mock.MockCategoryType; import com.arextest.schedule.bizlog.BizLogger; import com.arextest.schedule.common.CommonConstant; import com.arextest.schedule.common.SendSemaphoreLimiter; @@ -19,16 +18,11 @@ import com.arextest.schedule.model.ReplayActionCaseItem; import com.arextest.schedule.model.ReplayActionItem; import com.arextest.schedule.model.noiseidentify.ActionItemForNoiseIdentify; -import com.arextest.schedule.model.noiseidentify.ReplayNoiseDto; -import com.arextest.schedule.model.noiseidentify.ReplayNoiseItemDto; -import com.arextest.schedule.model.report.QueryNoiseResponseType; import com.arextest.schedule.progress.ProgressTracer; import com.arextest.schedule.sender.ReplaySender; import com.arextest.schedule.sender.ReplaySenderFactory; import com.arextest.schedule.service.MetricService; -import com.arextest.schedule.utils.MapUtils; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; @@ -173,81 +167,6 @@ public void rerunNoiseAnalysisRecovery(List actionItems) { replayNoiseRepository.removeReplayNoise(failedActions); } - @Override - public QueryNoiseResponseType queryNoise(String planId, String planItemId) { - - List appNoiseList = new ArrayList<>(); - - List replayNoiseDtoList = replayNoiseRepository.queryReplayNoise(planId, - planItemId); - Map> interfaceReplayNoiseMap = - replayNoiseDtoList.stream().collect(Collectors.groupingBy(ReplayNoiseDto::getOperationId)); - for (Map.Entry> interfaceReplayNoiseEntry : interfaceReplayNoiseMap.entrySet()) { - String operationId = interfaceReplayNoiseEntry.getKey(); - List interfaceReplayNoise = interfaceReplayNoiseEntry.getValue(); - if (CollectionUtils.isEmpty(interfaceReplayNoise)) { - continue; - } - - List randomNoiseList = new ArrayList<>(); - List disorderedArrayNoise = new ArrayList<>(); - - for (ReplayNoiseDto replayNoiseDto : interfaceReplayNoise) { - String categoryName = replayNoiseDto.getCategoryName(); - MockCategoryType mockCategoryType = MockCategoryType.create(categoryName); - - Map mayIgnoreItems = replayNoiseDto.getMayIgnoreItems(); - if (MapUtils.isNotEmpty(mayIgnoreItems)) { - Collection values = mayIgnoreItems.values(); - List randomNoiseItems = new ArrayList<>(values.size()); - for (ReplayNoiseItemDto itemDtoEntry : values) { - QueryNoiseResponseType.NoiseItem noiseItem = new QueryNoiseResponseType.NoiseItem(); - noiseItem.setNodeEntity(itemDtoEntry.getNodePath()); - noiseItem.setLogIndexes(itemDtoEntry.getLogIndexes()); - noiseItem.setCompareResultId(itemDtoEntry.getCompareResultId()); - randomNoiseItems.add(noiseItem); - } - if (CollectionUtils.isNotEmpty(randomNoiseItems)) { - randomNoiseList.add(new QueryNoiseResponseType.MockerNoiseItem(mockCategoryType, - replayNoiseDto.getOperationName(), categoryName, randomNoiseItems)); - } - } - - Map mayDisorderItems = replayNoiseDto.getMayDisorderItems(); - if (MapUtils.isNotEmpty(mayDisorderItems)) { - Collection values = mayDisorderItems.values(); - List disorderedArrayNoiseItems = new ArrayList<>( - values.size()); - for (ReplayNoiseItemDto itemDtoEntry : values) { - // XXX: simple array filter, improve: more accurate recommendations - if (itemDtoEntry.getPathCount() < 2) { - continue; - } - QueryNoiseResponseType.NoiseItem noiseItem = new QueryNoiseResponseType.NoiseItem(); - noiseItem.setNodeEntity(itemDtoEntry.getNodePath()); - noiseItem.setLogIndexes(itemDtoEntry.getLogIndexes()); - noiseItem.setCompareResultId(itemDtoEntry.getCompareResultId()); - disorderedArrayNoiseItems.add(noiseItem); - } - if (CollectionUtils.isNotEmpty(disorderedArrayNoiseItems)) { - disorderedArrayNoise.add(new QueryNoiseResponseType.MockerNoiseItem(mockCategoryType, - replayNoiseDto.getOperationName(), categoryName, disorderedArrayNoiseItems)); - } - } - } - - if (CollectionUtils.isNotEmpty(randomNoiseList) || CollectionUtils.isNotEmpty( - disorderedArrayNoise)) { - appNoiseList.add( - new QueryNoiseResponseType.InterfaceNoiseItem(operationId, randomNoiseList, - disorderedArrayNoise)); - } - } - QueryNoiseResponseType result = new QueryNoiseResponseType(); - result.setInterfaceNoiseItemList(appNoiseList); - return result; - } - private ActionItemForNoiseIdentify getActionItemForNoiseIdentify( MutablePair> itemPair, String contextName) { ReplayActionItem replayActionItem = itemPair.getLeft(); diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java index 8f256bd2..f051d0bb 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java @@ -1,6 +1,7 @@ package com.arextest.schedule.utils; import com.arextest.diff.model.log.NodeEntity; +import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; @@ -46,6 +47,19 @@ public static String getFuzzyPathStr(List path) { return sb.toString(); } + public static List getFuzzyPathStrList(List path) { + if (path == null || path.size() == 0) { + return null; + } + List list = new ArrayList<>(); + for (NodeEntity p : path) { + if (!StringUtils.isEmpty(p.getNodeName())) { + list.add(p.getNodeName()); + } + } + return list; + } + public static void removeLast(List list) { if (list == null || list.size() == 0) { diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/web/controller/ReplayNoiseController.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/web/controller/ReplayNoiseController.java index 66ddd975..35e57776 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/web/controller/ReplayNoiseController.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/web/controller/ReplayNoiseController.java @@ -1,31 +1,39 @@ package com.arextest.schedule.web.controller; import com.arextest.schedule.model.CommonResponse; -import com.arextest.schedule.model.report.QueryNoiseResponseType; -import com.arextest.schedule.service.noise.ReplayNoiseIdentify; +import com.arextest.schedule.model.noiseidentify.ExcludeNoiseRequestType; +import com.arextest.schedule.model.noiseidentify.QueryNoiseResponseType; +import com.arextest.schedule.service.noise.ReplayNoiseHandlerService; import javax.annotation.Resource; +import javax.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController -@RequestMapping(produces = {MediaType.APPLICATION_JSON_VALUE}) +@RequestMapping(path = "/api", produces = {MediaType.APPLICATION_JSON_VALUE}) public class ReplayNoiseController { @Resource - ReplayNoiseIdentify replayNoiseIdentify; + ReplayNoiseHandlerService replayNoiseHandlerService; - @GetMapping("/api/queryNoise") - @ResponseBody + @GetMapping("/queryNoise") public CommonResponse queryNoise(@RequestParam("planId") String planId, String planItemId) { - QueryNoiseResponseType queryNoiseResponseType = replayNoiseIdentify.queryNoise(planId, + QueryNoiseResponseType queryNoiseResponseType = replayNoiseHandlerService.queryNoise(planId, planItemId); return CommonResponse.successResponse("success", queryNoiseResponseType); } + @PostMapping(value = "/excludeNoise") + public CommonResponse excludeNoise(@RequestBody @Valid ExcludeNoiseRequestType excludeNoiseRequestType) { + boolean result = replayNoiseHandlerService.excludeNoise(excludeNoiseRequestType); + return CommonResponse.successResponse("success", result); + } + } diff --git a/arex-schedule-web-api/src/main/resources/application.properties b/arex-schedule-web-api/src/main/resources/application.properties index 99967d65..65d01825 100644 --- a/arex-schedule-web-api/src/main/resources/application.properties +++ b/arex-schedule-web-api/src/main/resources/application.properties @@ -15,6 +15,7 @@ arex.api.push.compareResult.url=${arex.api.service.api}/api/report/analyzeCompar arex.api.push.replayStatus.url=${arex.api.service.api}/api/report/pushReplayStatus arex.api.update.report.info.url=${arex.api.service.api}/api/report/updateReportInfo arex.api.remove.records.url=${arex.api.service.api}/api/report/removeRecordsAndScenes +arex.api.config.addExclusion.url=${arex.api.service.api}/api/config/comparison/exclusions/batchModify/INSERT #for schedule arex.api.config.system.url=${arex.api.service.api}/api/system/config/list arex.api.config.comparison.summary.url=${arex.api.service.api}/api/config/comparison/summary/queryByAppId/{appId} diff --git a/arex-schedule-web-api/src/main/resources/log4j2-dev.xml b/arex-schedule-web-api/src/main/resources/log4j2-dev.xml new file mode 100644 index 00000000..b5926f39 --- /dev/null +++ b/arex-schedule-web-api/src/main/resources/log4j2-dev.xml @@ -0,0 +1,37 @@ + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + + + + + + + + + + + + + From 94124dd61597d6b8f5a6d6ca9b8191ab37dc8fac Mon Sep 17 00:00:00 2001 From: rchen9 Date: Fri, 10 Nov 2023 13:16:56 +0800 Subject: [PATCH 2/2] feat: to avoid illegal char --- .../AsyncNoiseCaseAnalysisTaskRunnable.java | 6 ++--- .../arextest/schedule/utils/ListUtils.java | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/AsyncNoiseCaseAnalysisTaskRunnable.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/AsyncNoiseCaseAnalysisTaskRunnable.java index 84335fc0..4e32e59d 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/AsyncNoiseCaseAnalysisTaskRunnable.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/service/noise/AsyncNoiseCaseAnalysisTaskRunnable.java @@ -133,7 +133,7 @@ protected void doWithTracedRunning() { List nodePath = nodeEntry.getKey(); ReplayNoiseItemDto replayNoiseItemDto = nodeEntry.getValue(); replayNoiseItemDto.setNodePath(nodePath); - mayIgnoreItems.put(ListUtils.getFuzzyPathStr(nodePath), replayNoiseItemDto); + mayIgnoreItems.put(ListUtils.getFuzzyPathStrWithBase64(nodePath), replayNoiseItemDto); } replayNoiseDto.setMayIgnoreItems(mayIgnoreItems); } @@ -146,7 +146,7 @@ protected void doWithTracedRunning() { List nodePath = nodeEntry.getKey(); ReplayNoiseItemDto replayNoiseItemDto = nodeEntry.getValue(); replayNoiseItemDto.setNodePath(nodePath); - mayDisorderItems.put(ListUtils.getFuzzyPathStr(nodePath), replayNoiseItemDto); + mayDisorderItems.put(ListUtils.getFuzzyPathStrWithBase64(nodePath), replayNoiseItemDto); } replayNoiseDto.setMayDisorderItems(mayDisorderItems); } @@ -275,7 +275,7 @@ private void getAnalysisOfSingleCompareResult( LogEntity logEntity, int logIndex) { ReplayNoiseItemDto replayNoiseItemDto = singleAggContent.get(upperArrayNodePath); - String fuzzyPathStr = ListUtils.getFuzzyPathStr(logEntity.getPathPair().getLeftUnmatchedPath()); + String fuzzyPathStr = ListUtils.getFuzzyPathStrWithBase64(logEntity.getPathPair().getLeftUnmatchedPath()); if (replayNoiseItemDto == null) { replayNoiseItemDto = new ReplayNoiseItemDto(); replayNoiseItemDto.setCompareResult(replayCompareResult); diff --git a/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java b/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java index f051d0bb..6a8e626c 100644 --- a/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java +++ b/arex-schedule-web-api/src/main/java/com/arextest/schedule/utils/ListUtils.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; +import org.bson.internal.Base64; public class ListUtils { @@ -28,6 +29,27 @@ public static String convertPathToStringForShow(List nodes) { return path.toString(); } + /** + * calculate path without index for array, and encode with base64 to avoid illegal char + */ + public static String getFuzzyPathStrWithBase64(List path) { + if (path == null || path.size() == 0) { + return StringUtils.EMPTY; + } + StringBuilder sb = new StringBuilder(path.size() * 10); + for (NodeEntity p : path) { + if (!StringUtils.isEmpty(p.getNodeName())) { + if (sb.length() != 0) { + sb.append("\\"); + } + sb.append(p.getNodeName()); + } + } + return Base64.encode(sb.toString().getBytes()); + } + + + /** * calculate path without index for array */