Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache inline synapse expressions #2287

Merged
merged 3 commits into from
Jan 20, 2025
+141 −62
Merged

Cache inline synapse expressions #2287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c699634
Cache inline synapse expressions
SanojPunchihewa Jan 20, 2025
7edea4a
Update Freemarker Processor input to 'vars'
SanojPunchihewa Jan 20, 2025
79da6f7
Update Foreach mediator variables to 'vars'
SanojPunchihewa Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions modules/core/src/main/java/org/apache/synapse/config/xml/LogMediatorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import org.apache.axiom.om.OMAttribute;
import org.apache.axiom.om.OMElement;
import org.apache.synapse.Mediator;
import org.apache.synapse.SynapseException;
import org.apache.synapse.mediators.builtin.LogMediator;
import org.jaxen.JaxenException;

import javax.xml.namespace.QName;
import java.util.Properties;
Expand Down Expand Up @@ -71,8 +73,15 @@ public Mediator createSpecificMediator(OMElement elem, Properties properties) {
boolean containMessageTemplate = false;
OMElement messageElement = elem.getFirstChildWithName(ELEMENT_MESSAGE_Q);
if (messageElement != null && messageElement.getText() != null) {
logMediator.setMessageTemplate(messageElement.getText());
containMessageTemplate = true;
try {
containMessageTemplate = true;
logMediator.setMessageTemplate(messageElement.getText());
logMediator.processTemplateAndSetContentAware();
} catch (JaxenException e) {
String msg = "Invalid message template : " + e.getMessage();
log.error(msg);
throw new SynapseException(msg);
}
}

// Set the high level set of properties to be logged (i.e. log level)
Expand Down Expand Up @@ -126,7 +135,6 @@ public Mediator createSpecificMediator(OMElement elem, Properties properties) {
}

logMediator.addAllProperties(MediatorPropertyFactory.getMediatorProperties(elem));
logMediator.processTemplateAndSetContentAware();
addAllCommentChildrenToList(elem, logMediator.getCommentsList());

return logMediator;
Expand Down
32 changes: 18 additions & 14 deletions modules/core/src/main/java/org/apache/synapse/mediators/builtin/LogMediator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,14 @@
import org.apache.synapse.mediators.AbstractMediator;
import org.apache.synapse.mediators.MediatorProperty;
import org.apache.synapse.util.InlineExpressionUtil;
import org.apache.synapse.util.xpath.SynapseExpression;
import org.jaxen.JaxenException;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
* Logs the specified message into the configured logger. The log levels specify
Expand Down Expand Up @@ -80,6 +83,7 @@ public class LogMediator extends AbstractMediator {

private String messageTemplate = "";
private boolean isContentAware = false;
private final Map<String, SynapseExpression> inlineExpressionCache = new ConcurrentHashMap<>();

/**
* Logs the current message according to the supplied semantics
Expand Down Expand Up @@ -285,6 +289,12 @@ public void addProperty(MediatorProperty p) {

public void addAllProperties(List<MediatorProperty> list) {
properties.addAll(list);
for (MediatorProperty property : properties) {
if (property.getExpression() != null && property.getExpression().isContentAware()) {
isContentAware = true;
return;
}
}
}

public List<MediatorProperty> getProperties() {
Expand Down Expand Up @@ -324,7 +334,8 @@ private String trimLeadingSeparator(StringBuffer sb) {

private void processMessageTemplate(StringBuffer stringBuffer, MessageContext synCtx, String template) {
try {
stringBuffer.append(InlineExpressionUtil.processInLineSynapseExpressionTemplate(synCtx, template));
stringBuffer.append(InlineExpressionUtil.processInLineSynapseExpressionTemplate(synCtx, template,
inlineExpressionCache));
} catch (JaxenException e) {
handleException("Failed to process the message template : " + template, e, synCtx);
}
Expand All @@ -333,21 +344,14 @@ private void processMessageTemplate(StringBuffer stringBuffer, MessageContext sy
@Override
public boolean isContentAware() {

return isContentAware;
if (logLevel == MESSAGE_TEMPLATE || logLevel == CUSTOM) {
return isContentAware;
}
return true;
}

public void processTemplateAndSetContentAware() {
public void processTemplateAndSetContentAware() throws JaxenException {

if (logLevel == MESSAGE_TEMPLATE || logLevel == CUSTOM) {
for (MediatorProperty property : properties) {
if (property.getExpression() != null && property.getExpression().isContentAware()) {
isContentAware = true;
return;
}
}
isContentAware = InlineExpressionUtil.isInlineSynapseExpressionsContentAware(messageTemplate);
} else {
isContentAware = true;
}
isContentAware = InlineExpressionUtil.initInlineSynapseExpressions(messageTemplate, inlineExpressionCache);
}
}
2 changes: 1 addition & 1 deletion modules/core/src/main/java/org/apache/synapse/mediators/transform/pfutils/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ private Constants() {
public static final String CTX_PROPERTY_INJECTING_NAME = "ctx";
public static final String AXIS2_PROPERTY_INJECTING_NAME = "axis2";
public static final String TRANSPORT_PROPERTY_INJECTING_NAME = "trp";
public static final String VARIABLE_INJECTING_NAME = "var";
public static final String VARIABLE_INJECTING_NAME = "vars";
public static final String JSON_TYPE = "json";
public static final String XML_TYPE = "xml";
public static final String TEXT_TYPE = "text";
Expand Down
43 changes: 35 additions & 8 deletions .../src/main/java/org/apache/synapse/mediators/transform/pfutils/RegexTemplateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@
import org.apache.synapse.SynapseException;
import org.apache.synapse.commons.json.JsonUtil;
import org.apache.synapse.mediators.transform.ArgumentDetails;
import org.apache.synapse.util.InlineExpressionUtil;
import org.apache.synapse.util.xpath.SynapseExpression;
import org.jaxen.JaxenException;

import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.stream.XMLStreamException;
Expand All @@ -50,9 +52,10 @@ public class RegexTemplateProcessor extends TemplateProcessor {

private static final Log log = LogFactory.getLog(RegexTemplateProcessor.class);
// Pattern matches "${...}" (quoted), ${...} (unquoted), and $n
private final Pattern pattern = Pattern.compile("\"\\$\\{(.+?)\\}\"|\\$\\{(.+?)\\}|\\$(\\d+)");
private final Pattern pattern = Pattern.compile("\"\\$\\{([^}]+)\\}\"|\\$\\{([^}]+)\\}|\\$(\\d+)");

private final Gson gson = new Gson();
private final Map<String, SynapseExpression> inlineExpressionCache = new ConcurrentHashMap<>();

@Override
public String processTemplate(String template, String mediaType, MessageContext synCtx) {
Expand All @@ -63,7 +66,16 @@ public String processTemplate(String template, String mediaType, MessageContext
}

@Override
public void init() {
public void init() throws SynapseException {
String format = getFormat();
if (format != null) {
try {
InlineExpressionUtil.initInlineSynapseExpressions(format, inlineExpressionCache);
} catch (JaxenException e) {
String msg = "Invalid Payload format : " + e.getMessage();
throw new SynapseException(msg);
}
}
this.readInputFactoryProperties();
}

Expand All @@ -76,6 +88,7 @@ public void init() {
*/
private void replace(String format, StringBuffer result, String mediaType, MessageContext synCtx) {

Map<String, Object> inlineExpressionResults = new ConcurrentHashMap<>();
HashMap<String, ArgumentDetails>[] argValues = getArgValues(mediaType, synCtx);
HashMap<String, ArgumentDetails> replacement;
Map.Entry<String, ArgumentDetails> replacementEntry;
Expand All @@ -92,7 +105,7 @@ private void replace(String format, StringBuffer result, String mediaType, Messa
if (matcher.group(1) != null) {
// Handle "${...}" pattern (with quotes)
String expression = matcher.group(1);
Object expressionResult = evaluateExpression(expression, synCtx);
Object expressionResult = evaluateExpression(expression, synCtx, inlineExpressionResults);
if (expressionResult instanceof JsonPrimitive) {
replacementValue = prepareJSONPrimitiveReplacementValue(expressionResult, mediaType);
} else if (expressionResult instanceof JsonElement) {
Expand All @@ -118,7 +131,7 @@ private void replace(String format, StringBuffer result, String mediaType, Messa
} else if (matcher.group(2) != null) {
// Handle ${...} pattern (without quotes)
String expression = matcher.group(2);
Object expressionResult = evaluateExpression(expression, synCtx);
Object expressionResult = evaluateExpression(expression, synCtx, inlineExpressionResults);
replacementValue = expressionResult.toString();
if (expressionResult instanceof JsonPrimitive) {
replacementValue = prepareJSONPrimitiveReplacementValue(expressionResult, mediaType);
Expand Down Expand Up @@ -176,12 +189,26 @@ private String prepareJSONPrimitiveReplacementValue(Object expressionResult, Str
* @return evaluated result
* @throws JaxenException if an error occurs while evaluating the expression
*/
private Object evaluateExpression(String expression, MessageContext synCtx) throws JaxenException {
private Object evaluateExpression(String expression, MessageContext synCtx,
Map<String, Object> inlineExpressionResults) throws JaxenException {

if (expression.contains("xpath(")) {
return new SynapseExpression(expression).stringValueOf(synCtx);
SynapseExpression expressionObj = inlineExpressionCache.get(expression);
if (expressionObj == null) {
expressionObj = new SynapseExpression(expression);
inlineExpressionCache.put(expression, expressionObj);
}
if (inlineExpressionResults.containsKey(expression)) {
return inlineExpressionResults.get(expression);
} else {
Object result;
if (expression.contains("xpath(")) {
result = expressionObj.stringValueOf(synCtx);
} else {
result = expressionObj.objectValueOf(synCtx);
}
inlineExpressionResults.put(expression, result);
return result;
}
return new SynapseExpression(expression).objectValueOf(synCtx);
}

private String escapeJson(String value) {
Expand Down
3 changes: 2 additions & 1 deletion .../core/src/main/java/org/apache/synapse/mediators/transform/pfutils/TemplateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.synapse.MessageContext;
import org.apache.synapse.SynapseException;
import org.apache.synapse.commons.json.JsonUtil;
import org.apache.synapse.config.xml.SynapsePath;
import org.apache.synapse.mediators.transform.Argument;
Expand Down Expand Up @@ -90,7 +91,7 @@ public abstract class TemplateProcessor {
/**
* Execute pre-processing steps if needed
*/
public abstract void init();
public abstract void init() throws SynapseException;

/**
* Goes through SynapsePath argument list, evaluating each by calling stringValueOf and returns a HashMap String, String
Expand Down
22 changes: 13 additions & 9 deletions modules/core/src/main/java/org/apache/synapse/mediators/v2/ForEachMediatorV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.apache.synapse.mediators.eip.aggregator.ForEachAggregate;
import org.apache.synapse.transport.passthru.util.RelayUtils;
import org.apache.synapse.util.MessageHelper;
import org.apache.synapse.util.synapse.expression.constants.ExpressionConstants;
import org.apache.synapse.util.xpath.SynapseExpression;
import org.apache.synapse.util.xpath.SynapseExpressionUtils;
import org.apache.synapse.util.xpath.SynapseXPath;
Expand All @@ -81,6 +82,7 @@

public class ForEachMediatorV2 extends AbstractMediator implements ManagedLifecycle, FlowContinuableMediator {

public static final String VARIABLE_DOT = ExpressionConstants.VARIABLES + ".";
public static final String JSON_TYPE = "JSON";
public static final String XML_TYPE = "XML";
private final Object lock = new Object();
Expand Down Expand Up @@ -479,7 +481,7 @@ private void setJSONResultToVariable(JsonArray variable, ForEachAggregate aggreg
String[] msgSequence = prop.toString().split(EIPConstants.MESSAGE_SEQUENCE_DELEMITER);
JsonElement jsonElement = null;
try {
Object result = new SynapseExpression("payload").objectValueOf(synCtx);
Object result = new SynapseExpression(ExpressionConstants.PAYLOAD).objectValueOf(synCtx);
if (result instanceof JsonElement) {
jsonElement = (JsonElement) result;
}
Expand Down Expand Up @@ -513,7 +515,7 @@ private void updateOriginalPayload(MessageContext originalMessageContext, ForEac
Object prop = synCtx.getProperty(EIPConstants.MESSAGE_SEQUENCE + "." + id);
String[] msgSequence = prop.toString().split(EIPConstants.MESSAGE_SEQUENCE_DELEMITER);
JsonElement jsonElement = null;
Object result = new SynapseExpression("payload").objectValueOf(synCtx);
Object result = new SynapseExpression(ExpressionConstants.PAYLOAD).objectValueOf(synCtx);
if (result instanceof JsonElement) {
jsonElement = (JsonElement) result;
}
Expand Down Expand Up @@ -705,24 +707,26 @@ private String getVariableName(SynapsePath expression) {

private boolean isCollectionReferencedByVariable(SynapsePath expression) {

return expression.getExpression().startsWith("var.");
return expression.getExpression().startsWith(VARIABLE_DOT);
}

private JsonPath getJsonPathFromExpression(String expression) {

String jsonPath = expression;
if (jsonPath.startsWith("payload")) {
jsonPath = jsonPath.replace("payload", "$");
} else if (jsonPath.startsWith("var.")) {
// Remove the "var." prefix and variable name and replace it with "$" for JSON path
jsonPath = expression.replaceAll("var\\.\\w+\\.(\\w+)", "\\$.$1").replaceAll("var\\.\\w+", "\\$");
if (jsonPath.startsWith(ExpressionConstants.PAYLOAD)) {
jsonPath = jsonPath.replace(ExpressionConstants.PAYLOAD, ExpressionConstants.PAYLOAD_$);
} else if (jsonPath.startsWith(VARIABLE_DOT)) {
// Remove the "vars." prefix and variable name and replace it with "$" for JSON path
jsonPath = expression.replaceAll(ExpressionConstants.VARIABLES + "\\.\\w+\\.(\\w+)", "\\$.$1")
.replaceAll(ExpressionConstants.VARIABLES + "\\.\\w+", "\\$");
}
return JsonPath.compile(jsonPath);
}

private boolean isWholeContent(JsonPath jsonPath) {

return "$".equals(jsonPath.getPath().trim()) || "$.".equals(jsonPath.getPath().trim());
return ExpressionConstants.PAYLOAD_$.equals(jsonPath.getPath().trim())
|| ExpressionConstants.PAYLOAD_$.equals(jsonPath.getPath().trim());
}

public String getCounterVariable() {
Expand Down
36 changes: 24 additions & 12 deletions modules/core/src/main/java/org/apache/synapse/util/InlineExpressionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.synapse.util.xpath.SynapseXPath;
import org.jaxen.JaxenException;

import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.stream.XMLStreamException;
Expand Down Expand Up @@ -203,42 +204,53 @@ private static boolean isValidXML(String stringToValidate) {
}

/**
* Checks whether inline template contains content aware synapse expressions.
* Initialize the inline synapse expressions in the inline text and return whether the inline text is content aware
* Inline expressions will be denoted inside ${}
* e.g.: ${var.var1}, ${payload.element.id}
*
* @param inlineText Inline text string
* @param inlineText Inline text string
* @param expressionCache Cache to store the synapse expressions
* @return true if the string contains content aware inline synapse expressions, false otherwise
*/
public static boolean isInlineSynapseExpressionsContentAware(String inlineText) {
public static boolean initInlineSynapseExpressions(String inlineText, Map<String, SynapseExpression> expressionCache)
throws JaxenException {

boolean isContentAware = false;
Matcher matcher = SYNAPSE_EXPRESSION_PLACEHOLDER_PATTERN.matcher(inlineText);
while (matcher.find()) {
// Extract the expression inside ${...}
String expression = matcher.group(1);
if (SynapseExpressionUtils.isSynapseExpressionContentAware(expression)) {
return true;
// Extract the expression inside ${...} and add it to the cache
String placeholder = matcher.group(1);
SynapseExpression expression = new SynapseExpression(placeholder);
if (expression.isContentAware()) {
isContentAware = true;
}
expressionCache.put(placeholder, expression);
}
return false;
return isContentAware;
}

/**
* Process the inline template and replace the synapse expressions with the resolved values
*
* @param synCtx Message Context
* @param template Inline template
* @param synCtx Message Context
* @param template Inline template
* @param expressionCache Cache to store the synapse expressions
* @return Processed inline template
*/
public static String processInLineSynapseExpressionTemplate(MessageContext synCtx, String template)
public static String processInLineSynapseExpressionTemplate(MessageContext synCtx, String template,
Map<String, SynapseExpression> expressionCache)
throws JaxenException {

Matcher matcher = SYNAPSE_EXPRESSION_PLACEHOLDER_PATTERN.matcher(template);
StringBuffer result = new StringBuffer();
while (matcher.find()) {
// Extract the expression inside ${...}
String placeholder = matcher.group(1);
SynapseExpression expression = new SynapseExpression(placeholder);
SynapseExpression expression = expressionCache.get(placeholder);
if (expression == null) {
GDLMadushanka marked this conversation as resolved.
Show resolved Hide resolved
expression = new SynapseExpression(placeholder);
expressionCache.put(placeholder, expression);
}
String replacement = expression.stringValueOf(synCtx);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we calculate replacement values for the entire cached map at once. If we have the same expression used in several places, we don't have to evaluate again.

Copy link
Contributor Author

@SanojPunchihewa SanojPunchihewa Jan 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed with c699634 for the Payload factory. Logging the same expression multiple times in the same log is a rare cases. Hence skipped evaluated value caching in log mediator

matcher.appendReplacement(result, Matcher.quoteReplacement(replacement));
}
Expand Down
1 change: 1 addition & 0 deletions ...c/main/java/org/apache/synapse/util/synapse/expression/constants/ExpressionConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,5 @@ public class ExpressionConstants {
public static final String VAULT_LOOKUP = "wso2:vault-lookup('";
public static final String HEADERS = "headers";
public static final String ATTRIBUTES = "attributes";
public static final String VARIABLES = "vars";
}
Loading
Loading