change Ioc Type variable from enum to string to support custom ioc types

Signed-off-by: Surya Sashank Nistala <[email protected]>
opensearch-project · Jan 9, 2025 · 726da81 · 726da81
1 parent a7c5725
commit 726da81
Show file tree

Hide file tree

Showing 28 changed files with 5,489 additions and 392 deletions.
diff --git a/security-analytics-commons-1.0.0.jar b/security-analytics-commons-1.0.0.jar
diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java
@@ -12,14 +12,12 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.core.xcontent.XContentParserUtils;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.commons.model.STIX2;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Locale;
 
 /**
  * A data transfer object for STIX2IOC containing additional details.
@@ -58,7 +56,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
         }
 
         String name = null;
-        IOCType type = null;
+        String type = null;
         String value = null;
         String severity = null;
         Instant created = null;
@@ -89,7 +87,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
+                    type = xcp.text();
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();

diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java
@@ -11,22 +11,17 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.core.xcontent.XContentParserUtils;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.commons.model.STIX2;
-import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import org.opensearch.securityanalytics.util.XContentUtils;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 import java.util.UUID;
 
 public class STIX2IOC extends STIX2 implements Writeable, ToXContentObject {
@@ -46,7 +41,7 @@ public STIX2IOC() {
     public STIX2IOC(
             String id,
             String name,
-            IOCType type,
+            String type,
             String value,
             String severity,
             Instant created,
@@ -86,7 +81,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
         this(
                 sin.readString(), // id
                 sin.readString(), // name
-                new IOCType(sin.readString()), // type
+                sin.readString(), // type
                 sin.readString(), // value
                 sin.readString(), // severity
                 sin.readInstant(), // created
@@ -186,7 +181,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
         }
 
         String name = null;
-        IOCType type = null;
+        String type = null;
         String value = null;
         String severity = null;
         Instant created = null;
@@ -207,18 +202,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
                     name = xcp.text();
                     break;
                 case TYPE_FIELD:
-                    String typeString = xcp.text();
-                    try {
-                        type = new IOCType(typeString);
-                    } catch (Exception e) {
-                        String error = String.format(
-                                "Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ",
-                                typeString,
-                                id
-                        );
-                        logger.error(error, e);
-                        throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
-                    }
+                    type = xcp.text();
                     break;
                 case VALUE_FIELD:
                     value = xcp.text();
@@ -305,9 +289,6 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
     public void validate() throws IllegalArgumentException {
         if (super.getType() == null) {
             throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
-        } else if (!IOCType.supportedType(super.getType().toString())) {
-            logger.debug("Unsupported IOCType: {}", super.getType().toString());
-            throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
         }
 
         if (super.getValue() == null || super.getValue().isEmpty()) {

diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java
@@ -10,14 +10,11 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.core.xcontent.XContentParserUtils;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.commons.model.STIX2;
-import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -32,7 +29,7 @@ public class STIX2IOCDto implements Writeable, ToXContentObject {
 
     private String id;
     private String name;
-    private IOCType type;
+    private String type;
     private String value;
     private String severity;
     private Instant created;
@@ -50,7 +47,7 @@ public STIX2IOCDto() {}
     public STIX2IOCDto(
             String id,
             String name,
-            IOCType type,
+            String type,
             String value,
             String severity,
             Instant created,
@@ -149,7 +146,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
         }
 
         String name = null;
-        IOCType type = null;
+        String type = null;
         String value = null;
         String severity = null;
         Instant created = null;
@@ -180,18 +177,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    String typeString = xcp.text();
-                    try {
-                        type = new IOCType(typeString);
-                    } catch (Exception e) {
-                        String error = String.format(
-                                "Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ",
-                                typeString,
-                                id
-                        );
-                        logger.error(error, e);
-                        throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
-                    }
+                    type = xcp.text();
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();
@@ -286,11 +272,11 @@ public void setName(String name) {
         this.name = name;
     }
 
-    public IOCType getType() {
+    public String getType() {
         return type;
     }
 
-    public void setType(IOCType type) {
+    public void setType(String type) {
         this.type = type;
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
@@ -28,7 +28,6 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.securityanalytics.commons.model.IOC;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.commons.model.UpdateAction;
 import org.opensearch.securityanalytics.commons.store.FeedStore;
 import org.opensearch.securityanalytics.model.STIX2IOC;
@@ -224,7 +223,7 @@ private void initSourceConfigIndexes(StepListener<Void> stepListener) {
                     saTifSourceConfig.getIocTypes().forEach(type -> {
                         if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
                             DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
-                                    new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex);
+                                    new DefaultIocStoreConfig.IocToIndexDetails(type, iocIndexPattern, newActiveIndex);
                             ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails);
                         }
                     });

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java
@@ -373,7 +373,7 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SA
             STIX2IOC stix2IOC = new STIX2IOC(
                     UUID.randomUUID().toString(),
                     UUID.randomUUID().toString(),
-                    iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType),
+                    iocType,
                     iocValue,
                     "high",
                     now,

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java
@@ -11,7 +11,6 @@
 import org.opensearch.commons.alerting.model.Table;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 
 import java.io.IOException;
 import java.util.List;
@@ -63,19 +62,20 @@ public ActionRequestValidationException validate() {
         } else if (table.getSize() < 0 || table.getSize() > 10000) {
             validationException = ValidateActions
                     .addValidationError(String.format("size param must be between 0 and 10,000."), validationException);
-        } else {
-            for (String type : types) {
-                if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) {
-                    try {
-                        IOCType.fromString(type);
-                    } catch (IllegalArgumentException e) {
-                        validationException = ValidateActions
-                                .addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException);
-                        break;
-                    }
-                }
-            }
         }
+//        else {
+//            for (String type : types) {
+//                if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) {
+//                    try {
+//                        IOCType.fromString(type);
+//                    } catch (IllegalArgumentException e) {
+//                        validationException = ValidateActions
+//                                .addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException);
+//                        break;
+//                    }
+//                }
+//            }
+//        }
         return validationException;
     }
 

diff --git a/...in/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/...in/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java
@@ -5,7 +5,6 @@
 
 package org.opensearch.securityanalytics.threatIntel.common;
 
-import org.opensearch.securityanalytics.commons.model.IOCType;
 import org.opensearch.securityanalytics.threatIntel.model.CustomSchemaIocUploadSource;
 import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
 import org.opensearch.securityanalytics.threatIntel.model.S3Source;
@@ -52,16 +51,6 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
             errorMsgs.add("Source must not be empty");
         }
 
-        if (sourceConfigDto.getIocTypes() == null || sourceConfigDto.getIocTypes().isEmpty()) {
-            errorMsgs.add("Must specify at least one IOC type");
-        } else {
-            for (String s: sourceConfigDto.getIocTypes()) {
-                if (!IOCType.supportedType(s)) {
-                    errorMsgs.add("Invalid IOC type: " + s);
-                }
-            }
-        }
-
         if (sourceConfigDto.getType() == null) {
             errorMsgs.add("Type must not be empty");
         } else {

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java
@@ -9,7 +9,6 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.core.xcontent.XContentParserUtils;
-import org.opensearch.securityanalytics.commons.model.IOCType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -90,19 +89,19 @@ public static class IocToIndexDetails implements Writeable, ToXContent {
         public static final String IOC_TYPE_FIELD = "ioc_type";
         public static final String INDEX_PATTERN_FIELD = "index_pattern";
         public static final String ACTIVE_INDEX_FIELD = "active_index";
-        private final IOCType iocType;
+        private final String iocType;
         private final String indexPattern;
         private final String activeIndex;
 
-        public IocToIndexDetails(IOCType iocType, String indexPattern, String activeIndex) {
+        public IocToIndexDetails(String iocType, String indexPattern, String activeIndex) {
             this.iocType = iocType;
             this.indexPattern = indexPattern;
             this.activeIndex = activeIndex;
         }
 
         public IocToIndexDetails(StreamInput sin) throws IOException {
             this(
-                    new IOCType(sin.readString()),
+                    new String(sin.readString()),
                     sin.readString(),
                     sin.readString()
             );
@@ -124,7 +123,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
-            IOCType iocType = null;
+            String iocType = null;
             String indexPattern = null;
             String activeIndex = null;
 
@@ -135,7 +134,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
 
                 switch (fieldName) {
                     case IOC_TYPE_FIELD:
-                        iocType = toIocType(xcp.text());
+                        iocType = xcp.text();
                         break;
                     case INDEX_PATTERN_FIELD:
                         indexPattern = xcp.text();
@@ -150,16 +149,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
             return new IocToIndexDetails(iocType, indexPattern, activeIndex);
         }
 
-        public static IOCType toIocType(String name) {
-            try {
-                return new IOCType(name);
-            } catch (IllegalArgumentException e) {
-                log.error("Invalid Ioc type, cannot be parsed.", e);
-                return null;
-            }
-        }
-
-        public IOCType getIocType() {
+        public String getIocType() {
             return iocType;
         }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/JsonPathIocSchema.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/JsonPathIocSchema.java
@@ -16,7 +16,7 @@
  * Stores the schema defined by users who upload threat intelligence in a custom format.
  * Each field is defined and extracted using {@link com.jayway.jsonpath.JsonPath} annotation.
  * Each field is of type {@link JsonPathSchemaField}
- * If value of any given field is stored in format {"<key>": "<value>"}, then value of {@link JsonPathSchemaField#isKey()} field should be set as false.
+ * If value of any given field is stored in format  "key": "value", then value of {@link JsonPathSchemaField#isKey()} field should be set as false.
  * Else if value is stored in key itself, then value of {@link JsonPathSchemaField#isKey()} field should be set to true.
  */
 public class JsonPathIocSchema extends IocSchema<JsonPathIocSchema.JsonPathSchemaField> {

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java
@@ -167,7 +167,7 @@ public SATIFSourceConfigDto(StreamInput sin) throws IOException {
                 sin.readBoolean(), // is enabled
                 sin.readStringList(), // ioc types
                 sin.readBoolean(),
-                IocSchema.readFrom(sin)
+                sin.readBoolean() ? IocSchema.readFrom(sin) : null
         );
     }
 
@@ -207,7 +207,10 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeBoolean(enabledForScan);
         out.writeBoolean(iocSchema != null);
         if (iocSchema != null) {
+            out.writeBoolean(true);
             iocSchema.writeTo(out);
+        } else {
+            out.writeBoolean(false);
         }
     }