Api token authc/z implementation with Cache

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -132,6 +132,7 @@
 import org.opensearch.search.internal.SearchContext;
 import org.opensearch.search.query.QuerySearchResult;
 import org.opensearch.security.action.apitokens.ApiTokenAction;
+import org.opensearch.security.action.apitokens.ApiTokenIndexListenerCache;
 import org.opensearch.security.action.configupdate.ConfigUpdateAction;
 import org.opensearch.security.action.configupdate.TransportConfigUpdateAction;
 import org.opensearch.security.action.onbehalf.CreateOnBehalfOfTokenAction;
@@ -717,6 +718,15 @@
                     dlsFlsBaseContext
                 )
             );
+
+            // TODO: Is there a higher level approach that makes more sense here? Does this cover unsuccessful index ops?
+            if (ConfigConstants.OPENSEARCH_API_TOKENS_INDEX.equals(indexModule.getIndex().getName())) {
+                ApiTokenIndexListenerCache apiTokenIndexListenerCacher = ApiTokenIndexListenerCache.getInstance();
+                apiTokenIndexListenerCacher.initialize();
+                indexModule.addIndexOperationListener(apiTokenIndexListenerCacher);
+                log.warn("Security plugin started listening to operations on index {}", ConfigConstants.OPENSEARCH_API_TOKENS_INDEX);
+            }
+
             indexModule.forceQueryCacheProvider((indexSettings, nodeCache) -> new QueryCache() {
 
                 @Override

src/main/java/org/opensearch/security/action/apitokens/ApiTokenIndexListenerCache.java

@@ -0,0 +1,112 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.security.action.apitokens;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.engine.Engine;
+import org.opensearch.index.shard.IndexingOperationListener;
+
+/**
+ * This class implements an index operation listener for operations performed on api tokens
+ * These indices are defined on bootstrap and configured to listen in OpenSearchSecurityPlugin.java
+ */
+public class ApiTokenIndexListenerCache implements IndexingOperationListener {
+
+    private final static Logger log = LogManager.getLogger(ApiTokenIndexListenerCache.class);
+
+    private static final ApiTokenIndexListenerCache INSTANCE = new ApiTokenIndexListenerCache();
+    private final ConcurrentHashMap<String, String> idToJtiMap = new ConcurrentHashMap<>();
+
+    private Map<String, Permissions> jtis = new ConcurrentHashMap<>();
+
+    private boolean initialized;
+
+    private ApiTokenIndexListenerCache() {}
+
+    public static ApiTokenIndexListenerCache getInstance() {
+        return ApiTokenIndexListenerCache.INSTANCE;
+    }
+
+    /**
+     * Initializes the ApiTokenIndexListenerCache.
+     * This method is called during the plugin's initialization process.
+     *
+     */
+    public void initialize() {
+
+        if (initialized) {
+            return;
+        }
+
+        initialized = true;
+
+    }
+
+    public boolean isInitialized() {
+        return initialized;
+    }
+
+    /**
+     * This method is called after an index operation is performed.
+     * It adds the JTI of the indexed document to the cache and maps the document ID to the JTI (for deletion handling).
+     * @param shardId The shard ID of the index where the operation was performed.
+     * @param index The index where the operation was performed.
+     * @param result The result of the index operation.
+     */
+    @Override
+    public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+        BytesReference sourceRef = index.source();
+
+        try {
+            XContentParser parser = XContentType.JSON.xContent()
+                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, sourceRef.streamInput());
+
+            ApiToken token = ApiToken.fromXContent(parser);
+            jtis.put(token.getJti(), new Permissions(token.getClusterPermissions(), token.getIndexPermissions()));
+            idToJtiMap.put(index.id(), token.getJti());
+
+        } catch (IOException e) {
+            log.error("Failed to parse indexed document", e);
+        }
+    }
+
+    /**
+     * This method is called after a delete operation is performed.
+     * It deletes the corresponding document id in the map and the corresponding jti from the cache.
+     * @param shardId The shard ID of the index where the delete operation was performed.
+     * @param delete The delete operation that was performed.
+     * @param result The result of the delete operation.
+     */
+    @Override
+    public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+        String docId = delete.id();
+        String jti = idToJtiMap.remove(docId);
+        if (jti != null) {
+            jtis.remove(jti);
+            log.debug("Removed token with ID {} and JTI {} from cache", docId, jti);
+        }
+    }
+
+    public Map<String, Permissions> getJtis() {
+        return jtis;
+    }
+
+}

src/main/java/org/opensearch/security/action/apitokens/ApiTokenRepository.java

@@ -49,7 +49,7 @@ public String createApiToken(
     ) {
         apiTokenIndexHandler.createApiTokenIndexIfAbsent();
         // TODO: Add validation on whether user is creating a token with a subset of their permissions
-        ExpiringBearerAuthToken token = securityTokenManager.issueApiToken(name, expiration, clusterPermissions, indexPermissions);
+        ExpiringBearerAuthToken token = securityTokenManager.issueApiToken(name, expiration);
         ApiToken apiToken = new ApiToken(
             name,
             securityTokenManager.encryptToken(token.getCompleteToken()),

src/main/java/org/opensearch/security/action/apitokens/Permissions.java

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.security.action.apitokens;
+
+import java.util.List;
+
+public class Permissions {
+    private List<String> clusterPerm;
+    private List<ApiToken.IndexPermission> indexPermission;
+
+    // Constructor
+    public Permissions(List<String> clusterPerm, List<ApiToken.IndexPermission> indexPermission) {
+        this.clusterPerm = clusterPerm;
+        this.indexPermission = indexPermission;
+    }
+
+    // Getters and setters
+    public List<String> getClusterPerm() {
+        return clusterPerm;
+    }
+
+    public void setClusterPerm(List<String> clusterPerm) {
+        this.clusterPerm = clusterPerm;
+    }
+
+    public List<ApiToken.IndexPermission> getIndexPermission() {
+        return indexPermission;
+    }
+
+    public void setIndexPermission(List<ApiToken.IndexPermission> indexPermission) {
+        this.indexPermission = indexPermission;
+    }
+
+}

src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java

@@ -584,22 +584,24 @@
                     originalSource = "{}";
                 }
                 if (securityIndicesMatcher.test(shardId.getIndexName())) {
-                    try (
-                        XContentParser parser = XContentHelper.createParser(
-                            NamedXContentRegistry.EMPTY,
-                            THROW_UNSUPPORTED_OPERATION,
-                            originalResult.internalSourceRef(),
-                            XContentType.JSON
-                        )
-                    ) {
-                        Object base64 = parser.map().values().iterator().next();
-                        if (base64 instanceof String) {
-                            originalSource = (new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8));
-                        } else {
-                            originalSource = XContentHelper.convertToJson(originalResult.internalSourceRef(), false, XContentType.JSON);
+                    if (originalSource == null) {
+                        try (
+                            XContentParser parser = XContentHelper.createParser(
+                                NamedXContentRegistry.EMPTY,
+                                THROW_UNSUPPORTED_OPERATION,
+                                originalResult.internalSourceRef(),
+                                XContentType.JSON
+                            )
+                        ) {
+                            Object base64 = parser.map().values().iterator().next();
+                            if (base64 instanceof String) {
+                                originalSource = (new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8));
+                            } else {
+                                originalSource = XContentHelper.convertToJson(originalResult.internalSourceRef(), false, XContentType.JSON);
+                            }
+                        } catch (Exception e) {
+                            log.error(e.toString());
                         }
-                    } catch (Exception e) {
-                        log.error(e.toString());
                     }
 
                     try (
@@ -640,7 +642,7 @@
             }
         }
 
-        if (!complianceConfig.shouldLogWriteMetadataOnly()) {
+        if (!complianceConfig.shouldLogWriteMetadataOnly() && !complianceConfig.shouldLogDiffsForWrite()) {
             if (securityIndicesMatcher.test(shardId.getIndexName())) {
                 // current source, normally not null or empty
                 try (

src/main/java/org/opensearch/security/authtoken/jwt/JwtVendor.java

@@ -11,11 +11,9 @@
 
 package org.opensearch.security.authtoken.jwt;
 
-import java.io.IOException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.text.ParseException;
-import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Date;
 import java.util.List;
@@ -28,9 +26,6 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.security.action.apitokens.ApiToken;
 
 import com.nimbusds.jose.JOSEException;
 import com.nimbusds.jose.JWSAlgorithm;
@@ -157,14 +152,8 @@ public ExpiringBearerAuthToken createJwt(
     }
 
     @SuppressWarnings("removal")
-    public ExpiringBearerAuthToken createJwt(
-        final String issuer,
-        final String subject,
-        final String audience,
-        final long expiration,
-        final List<String> clusterPermissions,
-        final List<ApiToken.IndexPermission> indexPermissions
-    ) throws JOSEException, ParseException, IOException {
+    public ExpiringBearerAuthToken createJwt(final String issuer, final String subject, final String audience, final long expiration)
+        throws JOSEException, ParseException {
         final long currentTimeMs = timeProvider.getAsLong();
         final Date now = new Date(currentTimeMs);
 
@@ -178,20 +167,6 @@ public ExpiringBearerAuthToken createJwt(
         final Date expiryTime = new Date(expiration);
         claimsBuilder.expirationTime(expiryTime);
 
-        if (clusterPermissions != null) {
-            final String listOfClusterPermissions = String.join(",", clusterPermissions);
-            claimsBuilder.claim("cp", encryptString(listOfClusterPermissions));
-        }
-
-        if (indexPermissions != null) {
-            List<String> permissionStrings = new ArrayList<>();
-            for (ApiToken.IndexPermission permission : indexPermissions) {
-                permissionStrings.add(permission.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString());
-            }
-            final String listOfIndexPermissions = String.join(",", permissionStrings);
-            claimsBuilder.claim("ip", encryptString(listOfIndexPermissions));
-        }
-
         final JWSHeader header = new JWSHeader.Builder(JWSAlgorithm.parse(signingKey.getAlgorithm().getName())).build();
 
         final SignedJWT signedJwt = AccessController.doPrivileged(