From 0a2bd7a287efd9cde31906883f00f9fe516a5083 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Sun, 30 Apr 2023 09:01:42 +0200 Subject: [PATCH 01/23] Check array bounds --- .../natparse/parsing/AbstractParser.java | 18 +++++- .../natparse/parsing/DefineDataParser.java | 2 +- .../amshove/natparse/parsing/ParserError.java | 3 +- .../natparse/parsing/ParserErrors.java | 9 +++ .../SyntheticVariableStatementNode.java | 11 +++- .../natparse/parsing/SystemFunctionNode.java | 2 +- .../amshove/natparse/parsing/TypeChecker.java | 63 +++++++++++++++++++ .../amshove/natparse/parsing/ViewParser.java | 4 +- .../DefineDataParserDiagnosticTest.java | 2 +- .../parsing/DefineDataParserShould.java | 15 +++++ .../parsing/ResourceFolderBasedTest.java | 14 +++++ .../typing/writeWorkDynamicArrayBounds | 7 ++- .../src/main/resources/rules/NPP042 | 6 ++ 13 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 tools/ruletranslator/src/main/resources/rules/NPP042 diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index d86482df8..d3c79ea16 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -327,7 +327,21 @@ protected StatementNode identifierReference() throws ParseError } var node = symbolReferenceNode(token); - return new SyntheticVariableStatementNode(node); + var variableNode = new SyntheticVariableStatementNode(node); + if (peekKind(SyntaxKind.LPAREN) + && !peekKind(1, SyntaxKind.AD) + && !peekKind(1, SyntaxKind.EM)) + { + consumeMandatory(node, SyntaxKind.LPAREN); + variableNode.addDimension(consumeArrayAccess(variableNode)); + while (peekKind(SyntaxKind.COMMA)) + { + consume(variableNode); + variableNode.addDimension(consumeArrayAccess(variableNode)); + } + consumeMandatory(variableNode, SyntaxKind.RPAREN); + } + return variableNode; } protected FunctionCallNode functionCall(SyntaxToken token) throws ParseError @@ -758,7 +772,7 @@ protected IVariableReferenceNode consumeVariableReferenceNode(BaseSyntaxNode nod return reference; } - protected IOperandNode consumeArrayAccess(VariableReferenceNode reference) throws ParseError + protected IOperandNode consumeArrayAccess(BaseSyntaxNode reference) throws ParseError { if (peekKind(SyntaxKind.ASTERISK) && peekKind(1, SyntaxKind.RPAREN)) { diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java index 5041fe765..65849778b 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java @@ -801,7 +801,7 @@ private void addArrayDimension(VariableNode variable) throws ParseError throw new ParseError(peek()); } - while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN) && !peekKind(SyntaxKind.COMMA)) { var dimension = new ArrayDimension(); var lowerBound = extractArrayBound(new TokenNode(peek()), dimension); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java index 0ffe995e5..ffb238373 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java @@ -43,7 +43,8 @@ public enum ParserError INVALID_LITERAL_VALUE("NPP038"), REFERENCE_NOT_MUTABLE("NPP039"), UNSUPPORTED_PROGRAMMING_MODE("NPP040"), - INVALID_MODULE_TYPE("NPP041"); + INVALID_MODULE_TYPE("NPP041"), + INVALID_ARRAY_ACCESS("NPP042"); private final String id; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java index d65155c1f..6e739a1b8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java @@ -518,4 +518,13 @@ public static IDiagnostic invalidModuleType(String message, SyntaxToken errorTok ParserError.INVALID_MODULE_TYPE ); } + + public static IDiagnostic invalidArrayAccess(SyntaxToken token, String message) + { + return ParserDiagnostic.create( + message, + token, + ParserError.INVALID_ARRAY_ACCESS + ); + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java index 0042969e3..379c647cc 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java @@ -5,10 +5,14 @@ import org.amshove.natparse.lexing.SyntaxToken; import org.amshove.natparse.natural.*; +import java.util.ArrayList; +import java.util.List; + // TODO: Only exists until all statements are parse-able class SyntheticVariableStatementNode extends StatementNode implements IVariableReferenceNode { private final SymbolReferenceNode node; + private final List dimensions = new ArrayList<>(); public SyntheticVariableStatementNode(SymbolReferenceNode node) { @@ -66,6 +70,11 @@ public IPosition diagnosticPosition() @Override public ReadOnlyList dimensions() { - return ReadOnlyList.of(); // TODO: Do something + return ReadOnlyList.from(dimensions); + } + + void addDimension(IOperandNode dimension) + { + dimensions.add(dimension); } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java index 4d85f2984..d72c33a5e 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java @@ -10,7 +10,7 @@ class SystemFunctionNode extends BaseSyntaxNode implements ISystemFunctionNode { - private List parameter = new ArrayList<>(); + private final List parameter = new ArrayList<>(); private SyntaxKind systemFunction; @Override diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java index 02072099f..96bc4ab74 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java @@ -67,6 +67,69 @@ private void checkNode(ISyntaxNode node) { checkAlphanumericInitLength(typedVariableNode); } + + if (node instanceof IVariableReferenceNode variableReference) + { + checkVariableReference(variableReference); + } + } + + private void checkVariableReference(IVariableReferenceNode variableReference) + { + if (!(variableReference.reference()instanceof IVariableNode target)) + { + return; + } + + if (variableReference.dimensions().hasItems() && !target.isArray()) + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Using index access for a reference to non-array %s".formatted(target.name()) + ) + ); + } + + if (variableReference.dimensions().isEmpty() && target.isArray()) + { + if (!doesNotNeedDimensionInParentStatement(variableReference)) + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Missing index access, because %s is an array".formatted(target.name()) + ) + ); + } + } + + if (variableReference.dimensions().hasItems() && target.dimensions().hasItems() + && variableReference.dimensions().size() != target.dimensions().size()) + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Missing dimensions in array access. Got %d dimensions but %s has %d".formatted( + variableReference.dimensions().size(), + target.name(), + target.dimensions().size() + ) + ) + ); + } + } + + private boolean doesNotNeedDimensionInParentStatement(IVariableReferenceNode reference) + { + var parent = reference.parent(); + if (parent instanceof ISystemFunctionNode systemFunction) + { + var theFunction = systemFunction.systemFunction(); + return theFunction == SyntaxKind.OCC || theFunction == SyntaxKind.OCCURRENCE; + } + + return parent instanceof IExpandArrayNode || parent instanceof IReduceArrayNode || parent instanceof IResizeArrayNode; } private void checkWriteWork(IWriteWorkNode writeWork) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java index f6d56d812..51d061788 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java @@ -97,7 +97,7 @@ private VariableNode variable() throws ParseError if (peek().kind() == SyntaxKind.NUMBER_LITERAL || (peek().kind().isIdentifier() && isVariableDeclared(peek().symbolName()))) { - addArrayDimension(variable); + addArrayDimensions(variable); var typedDdmArrayVariable = typedVariableFromDdm(variable); consumeMandatory(typedDdmArrayVariable, SyntaxKind.RPAREN); return typedDdmArrayVariable; @@ -254,7 +254,7 @@ private void addArrayDimension(VariableNode variable) throws ParseError throw new ParseError(peek()); } - while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN) && !peekKind(SyntaxKind.COMMA)) { var dimension = new ArrayDimension(); var lowerBound = extractArrayBound(new TokenNode(peek()), dimension); diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java index 064a272eb..64281e242 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java @@ -8,6 +8,6 @@ class DefineDataParserDiagnosticTest extends ResourceFolderBasedTest @TestFactory Iterable diagnosticTests() { - return testFolder("definedatadiagnostics"); + return testFolder("definedatadiagnostics", "multiDimensionalViewArray"); } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index cb3997dbc..7059b9449 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -703,6 +703,21 @@ void parseAnArrayWithLowerAndUpperBoundBeingReferences() assertThat(theGroup.dimensions().first().upperBound()).isEqualTo(9); } + @Test + void parseTheNumberOfDimensionsCorrectly() + { + var defineData = assertParsesWithoutDiagnostics(""" + define data local + 1 #GROUP (1:2,1:120) + 2 #VAR (A10) + end-define + """); + + var group = assertNodeType(defineData.variables().first(), IGroupNode.class); + assertThat(group.dimensions()).hasSize(2); + assertThat(group.variables().first().dimensions()).hasSize(2); + } + @Test void addAReferenceToTheConstantForConstArrayDimension() { diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java index a6d2077a7..adcd91213 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java @@ -14,6 +14,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static org.assertj.core.api.Fail.fail; @@ -22,6 +23,14 @@ public abstract class ResourceFolderBasedTest { protected Iterable testFolder(String relativeFolderPath) + { + return testFolder(relativeFolderPath, null); + } + + /** + * Only runs the specified test. Pass null or omit to run all tests. + */ + protected Iterable testFolder(String relativeFolderPath, String testToRun) { var testFiles = ResourceHelper.findRelativeResourceFiles(relativeFolderPath, getClass()); @@ -36,6 +45,11 @@ protected Iterable testFolder(String relativeFolderPath) var testFilePath = Path.of(testFile); var testFileName = testFilePath.getFileName().toString(); + if (!testFileName.equals(testToRun)) + { + return Stream.of(); + } + var fileType = testFileName.contains(".") ? NaturalFileType.fromExtension(testFileName.split("\\.")[1]) : NaturalFileType.SUBPROGRAM; diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds index 7f59882dc..5de63e1a6 100644 --- a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds @@ -1,6 +1,7 @@ DEFINE DATA LOCAL 1 #ARR (A10/*) +1 #ARR3D (A10/*,*,*) 1 #I (I4) 1 #J (I4) END-DEFINE @@ -14,13 +15,13 @@ WRITE WORK FILE 1 #ARR(1:5) WRITE WORK FILE 1 #ARR(#I) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,#J) +WRITE WORK FILE 1 #ARR3D(2,#I,#J) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,#J) +WRITE WORK FILE 1 #ARR3D(2,#I,#J) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,*) +WRITE WORK FILE 1 #ARR3D(2,#I,*) /* This should raise no diagnostic, because it is no range access WRITE WORK FILE 1 #ARR(*) diff --git a/tools/ruletranslator/src/main/resources/rules/NPP042 b/tools/ruletranslator/src/main/resources/rules/NPP042 new file mode 100644 index 000000000..3f9af4ccc --- /dev/null +++ b/tools/ruletranslator/src/main/resources/rules/NPP042 @@ -0,0 +1,6 @@ +name: Invalid array access +priority: BLOCKER +tags: compile-time +type: BUG +description: +This reference to a variable either needs a dimension specified (because the target variable is an array) or needs toe have the dimension removed (because the target is not an array). From bd4cf343586b7ca710deae2cbb88b62cb96b5390 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Sun, 30 Apr 2023 10:22:51 +0200 Subject: [PATCH 02/23] Inherit array dimensions down to redefine childs --- .../natparse/parsing/DefineDataParser.java | 5 ++- .../natparse/parsing/RedefinitionNode.java | 4 +++ .../parsing/DefineDataParserShould.java | 32 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java index 65849778b..592bc7fb7 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java @@ -327,7 +327,10 @@ private GroupNode groupVariable(VariableNode variable) throws ParseError { for (var dimension : variable.dimensions()) { - groupNode.addDimension((ArrayDimension) dimension); + if (!groupNode.dimensions.contains(dimension)) + { + groupNode.addDimension((ArrayDimension) dimension); + } } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java index 19773d339..1c6e21ee4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java @@ -13,6 +13,10 @@ class RedefinitionNode extends GroupNode implements IRedefinitionNode public RedefinitionNode(VariableNode variable) { super(variable); + for (var inheritedDimension : variable.dimensions) + { + addDimension((ArrayDimension) inheritedDimension); + } } @Override diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index 7059b9449..0de808eba 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -812,6 +812,38 @@ void notReportALengthDiagnosticForNestedRedefineVariables() """); } + @Test + void inheritArrayDimensionsInNestedRedefines() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 01 UPPERVAR(A99) + 01 REDEFINE UPPERVAR + 02 #INNERGROUPARR(1:33) + 03 #INNERVAR(A2) + 03 REDEFINE #INNERVAR + 04 #CHAR1(A1) + 04 #CHAR2(A1) + 03 #ID(A1) + END-DEFINE + """); + + var firstRedefine = assertNodeType(defineData.variables().get(1), IRedefinitionNode.class); + var innerGroupArray = assertNodeType(firstRedefine.variables().first(), IGroupNode.class); + var firstVarInGroup = assertNodeType(innerGroupArray.variables().first(), ITypedVariableNode.class); + assertThat(firstVarInGroup.dimensions()).hasSize(1); + + var redefineOfFirstVarInGroup = assertNodeType(innerGroupArray.variables().get(1), IRedefinitionNode.class); + + assertThat(redefineOfFirstVarInGroup.variables().first().name()).isEqualTo("#CHAR1"); + assertThat(redefineOfFirstVarInGroup.variables().first().dimensions()).hasSize(1); // #CHAR1 + assertThat(redefineOfFirstVarInGroup.variables().last().name()).isEqualTo("#CHAR2"); + assertThat(redefineOfFirstVarInGroup.variables().last().dimensions()).hasSize(1); // #CHAR2 + + assertThat(innerGroupArray.variables().last().name()).isEqualTo("#ID"); + assertThat(innerGroupArray.variables().last().dimensions()).hasSize(1); + } + @Test void redefineGroups() { From 2476c770963867cd0857e3e9c726bb27292f4507 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Sun, 30 Apr 2023 19:32:03 +0200 Subject: [PATCH 03/23] Add an additional pass to correctly inherit array dimensions --- .../natparse/parsing/DefineDataParser.java | 27 +++++++++++++++++++ .../natparse/parsing/VariableNode.java | 15 +++++++++++ .../parsing/DefineDataParserShould.java | 18 +++++++++++++ 3 files changed, 60 insertions(+) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java index 592bc7fb7..27df4bacd 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java @@ -7,6 +7,7 @@ import org.amshove.natparse.natural.*; import org.amshove.natparse.natural.project.NaturalFileType; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -153,9 +154,35 @@ private ScopeNode scope() throws ParseError } } + passDownArrayDimensions(scopeNode); + return scopeNode; } + private void passDownArrayDimensions(ScopeNode scope) + { + for (var variable : scope.variables()) + { + inheritDimensions(variable); + } + } + + private void inheritDimensions(IVariableNode variable) + { + if (variable.parent()instanceof IVariableNode parentVariable && parentVariable.isArray()) + { + ((VariableNode) variable).inheritDimensions(parentVariable.dimensions()); + } + + if (variable instanceof GroupNode group) + { + for (var subVariable : group.variables()) + { + inheritDimensions(subVariable); + } + } + } + private UsingNode using() throws ParseError { var using = new UsingNode(); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java index f70faa7fa..324306c4d 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java @@ -8,6 +8,7 @@ import javax.annotation.Nonnull; import java.util.ArrayList; +import java.util.Collection; import java.util.List; class VariableNode extends BaseSyntaxNode implements IVariableNode @@ -141,4 +142,18 @@ void addDimension(ArrayDimension dimension) dimensions.add(dimension); addNode(dimension); } + + /** + * Inherits all the given dimensions if they're not specified for this variable yet. + */ + void inheritDimensions(ReadOnlyList dimensions) + { + for (var dimension : dimensions) + { + if (!this.dimensions.contains(dimension)) + { + this.dimensions.add(0, dimension); // add inhereted dimensions first, as they're defined first + } + } + } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index 0de808eba..b2a988590 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -844,6 +844,24 @@ void inheritArrayDimensionsInNestedRedefines() assertThat(innerGroupArray.variables().last().dimensions()).hasSize(1); } + @Test + void inheritArrayDimensionsInViews() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 MY-VIEW VIEW MY-DDM + 2 A-GROUP(1:12) + 3 A-VAR (N12,2) + END-DEFINE + """); + + var view = assertNodeType(defineData.variables().first(), IViewNode.class); + var group = assertNodeType(view.variables().first(), IGroupNode.class); + assertThat(group.dimensions()).hasSize(1); + var variable = assertNodeType(group.variables().first(), ITypedVariableNode.class); + assertThat(variable.dimensions()).hasSize(1); + } + @Test void redefineGroups() { From e5e2554d37516b858cbbf591c7b1fc4e67e8c3db Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Sun, 30 Apr 2023 20:05:37 +0200 Subject: [PATCH 04/23] Don't mistake attributes for array indices --- .../amshove/natparse/lexing/SyntaxKind.java | 6 ++++ .../natparse/parsing/AbstractParser.java | 20 +++++++++-- .../natparse/parsing/StatementListParser.java | 26 +++++++++++++- .../natparse/parsing/OperandParsingTests.java | 20 +++++++++++ .../parsing/ResourceFolderBasedTest.java | 2 +- .../natparse/parsing/typing/arrayAccessTests | 34 +++++++++++++++++++ 6 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index 10831f51e..a3018f585 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -2,6 +2,7 @@ public enum SyntaxKind { + EOF(false, false, false), // end of file LBRACKET(false, false, false), RBRACKET(false, false, false), LPAREN(false, false, false), @@ -758,4 +759,9 @@ public boolean canBeIdentifier() { return canBeIdentifier; } + + public boolean isAttribute() + { + return this == AD || this == DY || this == CD || this == EM; + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index d3c79ea16..92e03c73f 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -96,6 +96,19 @@ protected boolean peekKind(int offset, SyntaxKind kind) return !isAtEnd(offset) && peek(offset).kind() == kind; } + /** + * Returns the kind of the token at the given offset. If the offset is out of bounds, returns EOF. + */ + protected SyntaxKind getKind(int offset) + { + if (isAtEnd(offset)) + { + return SyntaxKind.EOF; + } + + return peek(offset).kind(); + } + protected boolean peekKind(SyntaxKind kind) { return peekKind(0, kind); @@ -756,7 +769,7 @@ protected IVariableReferenceNode consumeVariableReferenceNode(BaseSyntaxNode nod previousNode = reference; node.addNode(reference); - if (peekKind(SyntaxKind.LPAREN) && !peekKind(1, SyntaxKind.AD)) + if (peekKind(SyntaxKind.LPAREN) && !getKind(1).isAttribute() && !peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) { consumeMandatory(reference, SyntaxKind.LPAREN); reference.addDimension(consumeArrayAccess(reference)); @@ -846,7 +859,10 @@ protected void consumeAttributeDefinition(BaseSyntaxNode node) throws ParseError // this was built for CALLNAT, where a variable reference as parameter can have attribute definitions (only AD) // might be reusable for WRITE, DISPLAY, etc. for all kind of operands, but has to be fleshed out then consumeMandatory(node, SyntaxKind.LPAREN); - consumeMandatory(node, SyntaxKind.AD); + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + { + consume(node); + } consumeMandatory(node, SyntaxKind.RPAREN); } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index 3444a8332..70092fb82 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -265,6 +265,15 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case TERMINATE: statementList.addStatement(terminate()); break; + case LPAREN: + if (getKind(1).isAttribute()) + { + // Workaround for attributes. Should be added to the operand they belong to. + var tokenNode = new SyntheticTokenStatementNode(); + consumeAttributeDefinition(tokenNode); + statementList.addStatement(tokenNode); + break; + } case DECIDE: if (peekKind(1, SyntaxKind.FOR)) { @@ -1444,8 +1453,23 @@ private StatementNode write() throws ParseError consumeOptionally(write, SyntaxKind.NOTITLE); consumeOptionally(write, SyntaxKind.NOHDR); + while (!isAtEnd() && !isStatementStart()) + { + if (peekKind(SyntaxKind.LPAREN) && getKind(1).isAttribute()) + { + consumeAttributeDefinition(write); + } + else + { + if (!isOperand()) + { + break; + } + consumeOperandNode(write); + } + } - // TODO: Actual operands to WRITE not parsed + // TODO: Actual operands to WRITE not added as operands return write; } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java index 1694bed1a..63de9de71 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java @@ -468,4 +468,24 @@ void parseArrayIndicesWhichMightBeMistakenAsDecimalNumbers() var reference = assertIsVariableReference(operand, "#ARR"); assertThat(reference.dimensions()).hasSize(3); } + + @ParameterizedTest + @ValueSource(strings = + { + "DY='021I'01", "CD=RE", "AD=IO" + }) + void doNotParseAttributesAsArrayIndex(String parens) + { + var operand = parseOperand("#ARR(%s)".formatted(parens)); + var reference = assertIsVariableReference(operand, "#ARR"); + assertThat(reference.dimensions()).isEmpty(); + } + + @Test + void doNotParseLabelReferencesAsArrayIndex() + { + var operand = parseOperand("#ARR(R1.)"); + var reference = assertIsVariableReference(operand, "#ARR"); + assertThat(reference.dimensions()).isEmpty(); + } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java index adcd91213..5236d508c 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java @@ -45,7 +45,7 @@ protected Iterable testFolder(String relativeFolderPath, String tes var testFilePath = Path.of(testFile); var testFileName = testFilePath.getFileName().toString(); - if (!testFileName.equals(testToRun)) + if (testToRun != null && !testFileName.equals(testToRun)) { return Stream.of(); } diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests new file mode 100644 index 000000000..e2bf41f90 --- /dev/null +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests @@ -0,0 +1,34 @@ +DEFINE DATA +LOCAL +1 #NOT-ARRAY (A10) +1 #ARR (A10/*) +1 #I (I4) +END-DEFINE + +WRITE #NOT-ARRAY(DY='021I'01) /* No diagnostic, DY is an attribute +WRITE #NOT-ARRAY(HIS.) /* No diagnostic, label reference + +/* No diagnostics, array accessed +WRITE #ARR(*) +WRITE #ARR(5) + +/* No dimension specified +WRITE #ARR /* !{D:ERROR:NPP042} + +/* Shouldn't have a dimension +WRITE #NOT-ARRAY(5) /* !{D:ERROR:NPP042} + +/* Wrong dimensions specified +WRITE #ARR(*, *) /* !{D:ERROR:NPP042} +WRITE #ARR(5, 1) /* !{D:ERROR:NPP042} + +/* No diagnostic, *OCC needs actual array +#I := *OCC(#ARR) +#I := *OCCURRENCE(#ARR) + +/* No diagnostic, resizing statements need actual array +RESIZE ARRAY #ARR TO (1:10) +REDUCE ARRAY #ARR TO (1:5) +EXPAND ARRAY #ARR TO (1:8) + +END From 89ca3d97dc44d472c4578baeaf0399e2a51955de Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Sun, 30 Apr 2023 20:33:34 +0200 Subject: [PATCH 05/23] Add another case for attributes/label identifiers --- .../java/org/amshove/natparse/parsing/AbstractParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index 92e03c73f..1d3855829 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -342,8 +342,8 @@ protected StatementNode identifierReference() throws ParseError var node = symbolReferenceNode(token); var variableNode = new SyntheticVariableStatementNode(node); if (peekKind(SyntaxKind.LPAREN) - && !peekKind(1, SyntaxKind.AD) - && !peekKind(1, SyntaxKind.EM)) + && !getKind(1).isAttribute() + && !peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) { consumeMandatory(node, SyntaxKind.LPAREN); variableNode.addDimension(consumeArrayAccess(variableNode)); From aea798e33bc0dcc43105d93b46a0cbe19ffba456 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 08:54:44 +0200 Subject: [PATCH 06/23] Handle periodic groups, LBOUND, UBOUND and IF SPECIFIED --- .../amshove/natparse/lexing/SyntaxKind.java | 4 +- .../natparse/natural/IVariableNode.java | 2 + .../natural/builtin/BuiltInFunctionTable.java | 4 +- .../amshove/natparse/parsing/TypeChecker.java | 52 +++++++++++++++---- .../natparse/parsing/VariableNode.java | 8 +++ .../amshove/natparse/parsing/ViewNode.java | 6 +++ .../natparse/parsing/typing/arrayAccessTests | 25 +++++++++ 7 files changed, 87 insertions(+), 14 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index a3018f585..449038439 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -108,8 +108,8 @@ public enum SyntaxKind PROGRAM(true, true, false), ETID(false, true, false), CPU_TIME(false, true, false), - LBOUND(false, true, false), - UBOUND(false, true, false), + LBOUND(false, false, true), + UBOUND(false, false, true), SERVER_TYPE(false, true, false), // Kcheck reserved keywords diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java index 1517cbd28..bad01c0a8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java @@ -17,6 +17,8 @@ public non-sealed interface IVariableNode extends IReferencableNode, IParameterD ReadOnlyList dimensions(); + boolean isInView(); + default boolean isArray() { return dimensions() != null && dimensions().size() > 0; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java index 08ff74974..d58905a06 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java @@ -172,10 +172,10 @@ public class BuiltInFunctionTable unmodifiableVariable(SyntaxKind.INIT_PROGRAM, """ Return the name of program (transaction) currently executing as Natural. """, ALPHANUMERIC, 8), - unmodifiableVariable(SyntaxKind.LBOUND, """ + function(SyntaxKind.LBOUND, """ Returns the current lower boundary (index value) of an array for the specified dimension(s) (1, 2 or 3) or for all dimensions (asterisk (*) notation). """, INTEGER, 4), - unmodifiableVariable(SyntaxKind.UBOUND, """ + function(SyntaxKind.UBOUND, """ Returns the current upper boundary (index value) of an array for the specified dimension(s) (1, 2 or 3) or for all dimensions (asterisk (*) notation). """, INTEGER, 4), unmodifiableVariable(SyntaxKind.SERVER_TYPE, """ diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java index 96bc4ab74..806516683 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java @@ -7,6 +7,7 @@ import org.amshove.natparse.natural.builtin.BuiltInFunctionTable; import org.amshove.natparse.natural.builtin.IBuiltinFunctionDefinition; import org.amshove.natparse.natural.builtin.SystemVariableDefinition; +import org.amshove.natparse.natural.conditionals.ISpecifiedCriteriaNode; import java.util.ArrayList; import java.util.List; @@ -83,22 +84,26 @@ private void checkVariableReference(IVariableReferenceNode variableReference) if (variableReference.dimensions().hasItems() && !target.isArray()) { - diagnostics.add( - ParserErrors.invalidArrayAccess( - variableReference.referencingToken(), - "Using index access for a reference to non-array %s".formatted(target.name()) - ) - ); + if (!isPeriodicGroup(target))// periodic groups need to have their index specified. not allowed for "normal" groups + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Using index access for a reference to non-array %s".formatted(target.name()) + ) + ); + } } - if (variableReference.dimensions().isEmpty() && target.isArray()) + if (variableReference.dimensions().isEmpty() && (target.isArray() || isPeriodicGroup(target))) { if (!doesNotNeedDimensionInParentStatement(variableReference)) { + var message = isPeriodicGroup(target) ? "a periodic group" : "an array"; diagnostics.add( ParserErrors.invalidArrayAccess( variableReference.referencingToken(), - "Missing index access, because %s is an array".formatted(target.name()) + "Missing index access, because %s is %s".formatted(target.name(), message) ) ); } @@ -120,16 +125,43 @@ private void checkVariableReference(IVariableReferenceNode variableReference) } } + private boolean isPeriodicGroup(IVariableNode variable) + { + if (!(variable instanceof IGroupNode group)) + { + return false; + } + + if (!group.isInView()) + { + return false; + } + + var nextLevel = variable.level() + 1; + for (var periodicMember : group.variables()) + { + if (periodicMember.level() == nextLevel && !periodicMember.isArray()) + { + return false; + } + } + + return true; + } + private boolean doesNotNeedDimensionInParentStatement(IVariableReferenceNode reference) { var parent = reference.parent(); if (parent instanceof ISystemFunctionNode systemFunction) { var theFunction = systemFunction.systemFunction(); - return theFunction == SyntaxKind.OCC || theFunction == SyntaxKind.OCCURRENCE; + return theFunction == SyntaxKind.OCC || theFunction == SyntaxKind.OCCURRENCE || theFunction == SyntaxKind.UBOUND || theFunction == SyntaxKind.LBOUND; } - return parent instanceof IExpandArrayNode || parent instanceof IReduceArrayNode || parent instanceof IResizeArrayNode; + return parent instanceof IExpandArrayNode + || parent instanceof IReduceArrayNode + || parent instanceof IResizeArrayNode + || parent instanceof ISpecifiedCriteriaNode; } private void checkWriteWork(IWriteWorkNode writeWork) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java index 324306c4d..1b8afe3a8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java @@ -2,11 +2,13 @@ import org.amshove.natparse.IPosition; import org.amshove.natparse.NaturalParseException; +import org.amshove.natparse.NodeUtil; import org.amshove.natparse.ReadOnlyList; import org.amshove.natparse.lexing.SyntaxToken; import org.amshove.natparse.natural.*; import javax.annotation.Nonnull; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -119,6 +121,12 @@ public IPosition position() return declaration; } + @Override + public boolean isInView() + { + return NodeUtil.findFirstParentOfType(this, IViewNode.class) != null; + } + void setLevel(int level) { this.level = level; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java index 396a1fd25..987583bc9 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java @@ -31,6 +31,12 @@ public IDataDefinitionModule ddm() return ddm; } + @Override + public boolean isInView() + { + return true; + } + void setDdm(IDataDefinitionModule ddm) { this.ddm = ddm; diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests index e2bf41f90..f33f21fbc 100644 --- a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests @@ -1,8 +1,18 @@ DEFINE DATA +PARAMETER +1 #P-ARR (A40/1:*) OPTIONAL LOCAL 1 #NOT-ARRAY (A10) 1 #ARR (A10/*) +1 #ARR2 (A10/*) 1 #I (I4) +1 #GRP +2 #GRPARR1 (A10/*) +2 #GRPARR2 (A10/*) +1 THEVIEW VIEW OF THE-DDM +2 PERIODIC-GRP +3 INPER1 (A10/*) +3 INPER2 (A10/*) END-DEFINE WRITE #NOT-ARRAY(DY='021I'01) /* No diagnostic, DY is an attribute @@ -31,4 +41,19 @@ RESIZE ARRAY #ARR TO (1:10) REDUCE ARRAY #ARR TO (1:5) EXPAND ARRAY #ARR TO (1:8) +/* No array access allowed +WRITE #GRP(*) /* !{D:ERROR:NPP042} +/* Array access allowed because its a periodic group +WRITE PERIODIC-GRP(*) +/* Array access needed because its a periodic group +WRITE PERIODIC-GRP /* !{D:ERROR:NPP042} + +IF #P-ARR SPECIFIED /* IF SPECIFIED doesn't need to access index + IGNORE +END-IF + +/* Bounds need actual array, like OCC +#I := *UBOUND(#ARR2, 1) +#I := *LBOUND(#ARR2, 1) + END From 0cec2a1ae51c6840cae8b22072dea1f256474d9c Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 09:03:51 +0200 Subject: [PATCH 07/23] Do not treat the view itself as in a view --- .../src/main/java/org/amshove/natparse/parsing/ViewNode.java | 2 +- .../org/amshove/natparse/parsing/typing/arrayAccessTests | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java index 987583bc9..fcf72de7e 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java @@ -34,7 +34,7 @@ public IDataDefinitionModule ddm() @Override public boolean isInView() { - return true; + return false; } void setDdm(IDataDefinitionModule ddm) diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests index f33f21fbc..2a9230825 100644 --- a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests @@ -4,7 +4,6 @@ PARAMETER LOCAL 1 #NOT-ARRAY (A10) 1 #ARR (A10/*) -1 #ARR2 (A10/*) 1 #I (I4) 1 #GRP 2 #GRPARR1 (A10/*) @@ -53,7 +52,7 @@ IF #P-ARR SPECIFIED /* IF SPECIFIED doesn't need to access index END-IF /* Bounds need actual array, like OCC -#I := *UBOUND(#ARR2, 1) -#I := *LBOUND(#ARR2, 1) +#I := *UBOUND(#ARR, 1) +#I := *LBOUND(#ARR, 1) END From 5a6f728627da9539419d2a08568330cc5ead95a6 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 10:04:33 +0200 Subject: [PATCH 08/23] Refine array type definition lexing --- .../org/amshove/natparse/lexing/Lexer.java | 19 +++++- .../lexing/LexerForDatatypesShould.java | 62 +++++++++++++++++++ .../parsing/DefineDataParserShould.java | 15 +++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index 33a8dc210..549032ff4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -33,6 +33,7 @@ private enum LexerMode { DEFAULT, IN_DEFINE_DATA, + IN_DATA_TYPE } private LexerMode lexerMode = LexerMode.DEFAULT; @@ -55,7 +56,7 @@ public TokenList lex(String source, Path filePath) continue; } - if (consumeComment()) + if (lexerMode != LexerMode.IN_DATA_TYPE && consumeComment()) { continue; } @@ -73,10 +74,18 @@ public TokenList lex(String source, Path filePath) case '(': inParens = true; lastBeforeOpenParens = previous(); + if (lexerMode == LexerMode.IN_DEFINE_DATA && previous().kind() != SyntaxKind.LESSER_SIGN) + { + lexerMode = LexerMode.IN_DATA_TYPE; + } createAndAddCurrentSingleToken(SyntaxKind.LPAREN); continue; case ')': inParens = false; + if (lexerMode == LexerMode.IN_DATA_TYPE) + { + lexerMode = LexerMode.IN_DEFINE_DATA; + } lastBeforeOpenParens = null; createAndAddCurrentSingleToken(SyntaxKind.RPAREN); continue; @@ -767,7 +776,7 @@ private void consumeIdentifier() if (scanner.peek() == '/' && scanner.peek(1) == '*') { // Slash is a valid character for identifiers, but an asterisk is not. - // If a variable is named #MYVAR/* we can safely assume its a variable followed + // If a variable is named #MYVAR/* we can safely assume it's a variable followed // by a comment. break; } @@ -900,6 +909,12 @@ private void consumeIdentifierOrKeyword() break; } + if (scanner.peek() == '/' && lexerMode == LexerMode.IN_DATA_TYPE) + { + // Slash is a valid character for identifiers, if we're lexing a datatype we can be pretty confident about the slash being for array dimensions + break; + } + if (scanner.peek() == '/') { // TODO: this does not work for "KEYWORD/*", eg. END-SUBROUTINE/* bla bla kindHint = SyntaxKind.IDENTIFIER; diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java new file mode 100644 index 000000000..710bc7295 --- /dev/null +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java @@ -0,0 +1,62 @@ +package org.amshove.natparse.lexing; + +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +class LexerForDatatypesShould extends AbstractLexerTest +{ + @Test + void recognizeSimpleDataTypes() + { + assertTokens("A10", SyntaxKind.IDENTIFIER); + } + + @Test + void recognizeArrays() + { + assertTokens("A10/*", SyntaxKind.IDENTIFIER, SyntaxKind.SLASH, SyntaxKind.ASTERISK); + } + + @Test + void recognizeArraysWithWhitespace() + { + assertTokens("A10/ 1: 10", SyntaxKind.IDENTIFIER, SyntaxKind.SLASH, SyntaxKind.NUMBER_LITERAL, SyntaxKind.COLON, SyntaxKind.NUMBER_LITERAL); + } + + @Override + protected void assertTokens(String source, SyntaxKind... expectedKinds) + { + var tokens = lexSource(""" + DEFINE DATA LOCAL + 1 #VAR (%s) + END-DEFINE + """.formatted(source)); + + while (tokens.peek().kind() != SyntaxKind.LPAREN) + { + tokens.advance(); + } + tokens.advance(); + + for (var expectedKind : expectedKinds) + { + var actualToken = tokens.advance(); + assertThat(actualToken.kind()) + .as("Tokens from here to end: " + tokens.subrange(tokens.getCurrentOffset(), tokens.size() - 1).stream().map(SyntaxToken::kind).map(Enum::name).collect(Collectors.joining(", "))) + .isEqualTo(expectedKind); + } + + assertThat(tokens.peek().kind()) + .as("Expected data type end token (RPAREN)") + .isEqualTo(SyntaxKind.RPAREN); + } +} diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index b2a988590..2e0f63e80 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -1108,6 +1108,21 @@ void allowVAsUnboundInParameterScope() assertThat(variable.dimensions().first().isUpperUnbound()).isTrue(); } + @Test + void parseArrayBoundsWithNastyWhitespace() + { + var defineData = assertParsesWithoutDiagnostics(""" + define data + parameter + 1 #p-array (A3/ 1: 50) + end-define + """); + + var variable = findVariable(defineData, "#p-array", ITypedVariableNode.class); + assertThat(variable.dimensions().first().lowerBound()).isEqualTo(1); + assertThat(variable.dimensions().first().upperBound()).isEqualTo(50); + } + @Test void parseArrayInitialValues() { From 0e0428acdbbfbfe3661a75107d7aa11826a3f47f Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 10:28:09 +0200 Subject: [PATCH 09/23] Losely parse DEFINE PROTOTYPE --- docs/implemented-statements.md | 4 +- .../natural/IDefinePrototypeNode.java | 22 +++++++++ .../natparse/parsing/DefinePrototypeNode.java | 44 ++++++++++++++++++ .../natparse/parsing/StatementListParser.java | 46 ++++++++++++++++++- .../parsing/StatementListParserShould.java | 26 +++++++++++ 5 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java create mode 100644 libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java diff --git a/docs/implemented-statements.md b/docs/implemented-statements.md index ad591971d..32cb937f2 100644 --- a/docs/implemented-statements.md +++ b/docs/implemented-statements.md @@ -41,7 +41,7 @@ partial - partially implemented to prevent false positives (6) | DEFINE DATA | :white_check_mark: | | DEFINE FUNCTION | partial | | DEFINE PRINTER | :white_check_mark: | -| DEFINE PROTOTYPE | :x: | +| DEFINE PROTOTYPE | partial | | DEFINE SUBROUTINE | :white_check_mark: | | DEFINE WINDOW | partial | | DEFINE WORK FILE | :white_check_mark: | @@ -55,7 +55,7 @@ partial - partially implemented to prevent false positives (6) | END TRANSACTION | :x: | | ESCAPE | :white_check_mark: | | EXAMINE | :white_check_mark: | -| EXPAND | :white_check_mark: | +| EXPAND | :white_check_mark: | | FETCH | :white_check_mark: | | FIND | :white_check_mark: | | FOR | :white_check_mark: | diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java new file mode 100644 index 000000000..203c95c76 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java @@ -0,0 +1,22 @@ +package org.amshove.natparse.natural; + +import org.amshove.natparse.lexing.SyntaxToken; + +import javax.annotation.Nullable; + +public interface IDefinePrototypeNode extends IStatementNode +{ + /** + * Contains the name of the prototype. Could be a reference to an external function or a variable reference if + * {@code isVariable()} is true. + */ + SyntaxToken nameToken(); + + boolean isVariable(); + + /** + * The reference to the variable containing the function name. Is filled if {@code isVariable()} is true. + */ + @Nullable + IVariableReferenceNode variableReference(); +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java new file mode 100644 index 000000000..df1a400a2 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java @@ -0,0 +1,44 @@ +package org.amshove.natparse.parsing; + +import org.amshove.natparse.lexing.SyntaxToken; +import org.amshove.natparse.natural.IDefinePrototypeNode; +import org.amshove.natparse.natural.IOperandNode; +import org.amshove.natparse.natural.IVariableReferenceNode; + +import javax.annotation.Nullable; + +class DefinePrototypeNode extends StatementNode implements IDefinePrototypeNode +{ + + private SyntaxToken prototypeName; + private IVariableReferenceNode variableReference; + + @Override + public SyntaxToken nameToken() + { + return variableReference != null ? variableReference.referencingToken() : prototypeName; + } + + @Override + public boolean isVariable() + { + return variableReference != null; + } + + @Nullable + @Override + public IVariableReferenceNode variableReference() + { + return variableReference; + } + + void setPrototype(SyntaxToken token) + { + prototypeName = token; + } + + void setVariableReference(IVariableReferenceNode reference) + { + this.variableReference = reference; + } +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index 70092fb82..a636879a1 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -211,8 +211,10 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case PRINTER -> statementList.addStatement(definePrinter()); case WINDOW -> statementList.addStatement(defineWindow()); case WORK -> statementList.addStatement(defineWork()); - case PROTOTYPE, DATA -> - { // not implemented statements. DATA needs to be handled when parsing functions and external subroutines + case PROTOTYPE -> statementList.addStatement(definePrototype()); + case DATA -> + { + // can this even happen? tokens.advance(); tokens.advance(); } @@ -351,6 +353,46 @@ private StatementListNode statementList(SyntaxKind endTokenKind) return statementList; } + private StatementNode definePrototype() throws ParseError + { + if (peekKind(2, SyntaxKind.FOR) || peekKind(2, SyntaxKind.VARIABLE)) + { + return definePrototypeVariable(); + } + + var prototype = new DefinePrototypeNode(); + var opening = consumeMandatory(prototype, SyntaxKind.DEFINE); + consumeMandatory(prototype, SyntaxKind.PROTOTYPE); + + var name = consumeMandatoryIdentifier(prototype); // TODO: Sideload + prototype.setPrototype(name); + while (!isAtEnd() && !peekKind(SyntaxKind.END_PROTOTYPE)) + { + consume(prototype); // incomplete + } + + consumeMandatoryClosing(prototype, SyntaxKind.END_PROTOTYPE, opening); + return prototype; + } + + private StatementNode definePrototypeVariable() throws ParseError + { + var prototype = new DefinePrototypeNode(); + var opening = consumeMandatory(prototype, SyntaxKind.DEFINE); + consumeMandatory(prototype, SyntaxKind.PROTOTYPE); + consumeOptionally(prototype, SyntaxKind.FOR); + consumeMandatory(prototype, SyntaxKind.VARIABLE); + + prototype.setVariableReference(consumeVariableReferenceNode(prototype)); + while (!isAtEnd() && !peekKind(SyntaxKind.END_PROTOTYPE)) + { + consume(prototype); // incomplete + } + + consumeMandatoryClosing(prototype, SyntaxKind.END_PROTOTYPE, opening); + return prototype; + } + private StatementNode terminate() throws ParseError { var terminate = new TerminateNode(); diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index 095c93883..d989dc294 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1894,4 +1894,30 @@ void parseReduceDynamicWithErrorNr() var stmt = assertParsesSingleStatement("REDUCE DYNAMIC #VAR TO 20 GIVING #ERR", IReduceDynamicNode.class); assertIsVariableReference(stmt.errorVariable(), "#ERR"); } + + @Test + void parseDefinePrototype() + { + var prototype = assertParsesSingleStatement(""" + DEFINE PROTOTYPE HI RETURNS (L) + END-PROTOTYPE + """, IDefinePrototypeNode.class); + + assertThat(prototype.nameToken().symbolName()).isEqualTo("HI"); + assertThat(prototype.isVariable()).isFalse(); + assertThat(prototype.variableReference()).isNull(); + } + + @Test + void parseDefinePrototypeVariable() + { + var prototype = assertParsesSingleStatement(""" + DEFINE PROTOTYPE VARIABLE HI RETURNS (L) + END-PROTOTYPE + """, IDefinePrototypeNode.class); + + assertThat(prototype.nameToken().symbolName()).isEqualTo("HI"); + assertThat(prototype.isVariable()).isTrue(); + assertThat(prototype.variableReference()).isNotNull(); + } } From 02a4127563a7066a84ee909f38ac99712d04a7cc Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 10:31:58 +0200 Subject: [PATCH 10/23] Update docs --- docs/implemented-statements.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/implemented-statements.md b/docs/implemented-statements.md index 32cb937f2..d3514d613 100644 --- a/docs/implemented-statements.md +++ b/docs/implemented-statements.md @@ -4,11 +4,11 @@ This document tracks the implementation status of Natural statements. Legend: -:x: - not implemented (65) +:x: - not implemented (64) :white_check_mark: - implemented or reporting (51) -partial - partially implemented to prevent false positives (6) +partial - partially implemented to prevent false positives (7) | Statement | Status | | --- |--------------------| From af3e43ef44769dcdebdca59ee7c2409f05c7f838 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 10:43:39 +0200 Subject: [PATCH 11/23] Fix array dimension parsing for math expressions --- .../natparse/parsing/DefineDataParser.java | 5 +++- .../parsing/DefineDataParserShould.java | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java index 27df4bacd..3fe17140c 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java @@ -7,7 +7,6 @@ import org.amshove.natparse.natural.*; import org.amshove.natparse.natural.project.NaturalFileType; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -857,6 +856,10 @@ private void addArrayDimension(VariableNode variable) throws ParseError dimension.setLowerBound(lowerBound); dimension.setUpperBound(upperBound); variable.addDimension(dimension); + while (!isAtEnd() && !peekKind(SyntaxKind.COMMA) && !peekKind(SyntaxKind.RPAREN)) + { + consume(dimension); + } } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index 2e0f63e80..bdfa77d0f 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -862,6 +862,34 @@ void inheritArrayDimensionsInViews() assertThat(variable.dimensions()).hasSize(1); } + @Test + void parseTheCorrectNumberOfDimensionsForTypesWithMath() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 #VAR (N12) CONST<5> + 1 #ARR (A10/1:#VAR+1) + END-DEFINE + """); + + var array = findVariable(defineData, "#ARR", ITypedVariableNode.class); + assertThat(array.dimensions()).hasSize(1); + } + + @Test + void parseTheCorrectNumberOfDimensionsForTypesWithMathAndComma() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 #VAR (N12) CONST<5> + 1 #ARR (A10/1:#VAR+ 1,1 :20) + END-DEFINE + """); + + var array = findVariable(defineData, "#ARR", ITypedVariableNode.class); + assertThat(array.dimensions()).hasSize(2); + } + @Test void redefineGroups() { From 87bc166029babc838d87f5edec57a818f6e70219 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 11:00:28 +0200 Subject: [PATCH 12/23] Add some more attributes --- .../org/amshove/natparse/lexing/Lexer.java | 94 ++++++++++++++++++- .../amshove/natparse/lexing/SyntaxKind.java | 2 +- .../LexerForAttributeControlsShould.java | 55 +++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index 549032ff4..64c19d2c7 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -858,7 +858,37 @@ private void consumeIdentifierOrKeyword() if (inParens && scanner.peekText("DY=")) { - dynamicAttribte(); + dynamicAttribute(); + return; + } + + if (inParens && scanner.peekText("NL=")) + { + numericLengthAttribute(); + return; + } + + if (inParens && scanner.peekText("AL=")) + { + alphanumericLengthAttribute(); + return; + } + + if (inParens && scanner.peekText("DF=")) + { + dateFormatAttribute(); + return; + } + + if (inParens && scanner.peekText("IP=")) + { + inputPromptAttribute(); + return; + } + + if (inParens && scanner.peekText("IS=")) + { + identicalSuppressAttribute(); return; } @@ -1045,7 +1075,67 @@ private void attributeDefinition() createAndAdd(SyntaxKind.AD); } - private void dynamicAttribte() + private void alphanumericLengthAttribute() + { + scanner.start(); + scanner.advance(3); // AL= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.AL); + } + + private void dateFormatAttribute() + { + scanner.start(); + scanner.advance(3); // DF= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.DF); + } + + private void inputPromptAttribute() + { + scanner.start(); + scanner.advance(3); // IP= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.IP); + } + + private void identicalSuppressAttribute() + { + scanner.start(); + scanner.advance(3); // IS= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.IS); + } + + private void numericLengthAttribute() + { + scanner.start(); + scanner.advance(3); // NL= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.NL); + } + + private void dynamicAttribute() { scanner.start(); scanner.advance(3); // DY= diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index 449038439..e7e366589 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -762,6 +762,6 @@ public boolean canBeIdentifier() public boolean isAttribute() { - return this == AD || this == DY || this == CD || this == EM; + return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS; } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java index 0e927d3dd..8fa3687c0 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java @@ -106,4 +106,59 @@ void consumeComplexDynamicAttributes() token(SyntaxKind.RPAREN) ); } + + @Test + void consumeNL() + { + assertTokens( + "(NL=12,7)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.NL, "NL=12,7"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeAL() + { + assertTokens( + "(AL=20)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.AL, "AL=20"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeDF() + { + assertTokens( + "(DF=S)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.DF, "DF=S"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeIP() + { + assertTokens( + "(IP=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.IP, "IP=OFF"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeIS() + { + assertTokens( + "(IS=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.IS, "IS=OFF"), + token(SyntaxKind.RPAREN) + ); + } } From 6e19f3c66d894e9f9680e9b744a4106eb2d1fbd6 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 11:17:48 +0200 Subject: [PATCH 13/23] Consume some more WRITE attributes --- .../natparse/parsing/StatementListParser.java | 11 ++++++++--- .../natparse/parsing/StatementListParserShould.java | 2 +- .../regressiontests/writeThinksFlagsAreVariables | 12 ++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index a636879a1..045a73760 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -1471,6 +1471,8 @@ private StatementNode display() throws ParseError return display; } + private static final Set OPTIONAL_WRITE_FLAGS = Set.of(SyntaxKind.NOTITLE, SyntaxKind.NOHDR, SyntaxKind.USING, SyntaxKind.MAP, SyntaxKind.FORM, SyntaxKind.TITLE, SyntaxKind.LEFT, SyntaxKind.JUSTIFIED, SyntaxKind.UNDERLINED); + private StatementNode write() throws ParseError { var write = new WriteNode(); @@ -1493,8 +1495,10 @@ private StatementNode write() throws ParseError consumeMandatory(write, SyntaxKind.RPAREN); } - consumeOptionally(write, SyntaxKind.NOTITLE); - consumeOptionally(write, SyntaxKind.NOHDR); + while (consumeAnyOptionally(write, OPTIONAL_WRITE_FLAGS)) + { + // advances automatically + } while (!isAtEnd() && !isStatementStart()) { if (peekKind(SyntaxKind.LPAREN) && getKind(1).isAttribute()) @@ -1503,7 +1507,8 @@ private StatementNode write() throws ParseError } else { - if (!isOperand()) + if ((consumeOptionally(write, SyntaxKind.NO) && consumeOptionally(write, SyntaxKind.PARAMETER)) + || !isOperand()) { break; } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index d989dc294..fa3e1ff64 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1011,7 +1011,7 @@ void parseWriteWithReportSpecification() void parseWriteWithAttributeDefinition() { var write = assertParsesSingleStatement("WRITE (AD=UL AL=17 NL=8)", IWriteNode.class); - assertThat(write.descendants()).hasSize(10); + assertThat(write.descendants()).hasSize(6); } @Test diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables new file mode 100644 index 000000000..d2374941a --- /dev/null +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables @@ -0,0 +1,12 @@ +DEFINE DATA +LOCAL +END-DEFINE + +/* These should all not raise any diagnostics, because no variables are referenced +WRITE NOTITLE NOHDR USING FORM 'DAFORM' +WRITE NOTITLE NOHDR USING MAP 'DAMAP' +WRITE NOTITLE NOHDR USING MAP 'DAMAP' NO PARAMETER +WRITE (4) 'HI' +WRITE TITLE LEFT JUSTIFIED UNDERLINED 'Hi' SKIP 2 LINES + +END From 0ef5102069f1fa9285a1e959180bf82883478905 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 11:50:43 +0200 Subject: [PATCH 14/23] Parse CV --- .../definition/DefinitionEndpointTests.java | 17 +++++++++++++++++ .../java/org/amshove/natparse/lexing/Lexer.java | 14 ++++++++++++++ .../org/amshove/natparse/lexing/SyntaxKind.java | 2 +- .../natparse/parsing/AbstractParser.java | 9 ++++++++- .../lexing/LexerForAttributeControlsShould.java | 12 ++++++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) diff --git a/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java b/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java index f0636e2d8..6217e5640 100644 --- a/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java +++ b/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java @@ -202,6 +202,23 @@ void definitionsShouldReturnTheDefinitionOfVariablesInAssignments() ); } + @Test + void definitionsShouldReturnTheDefinitionOfVariablesInCVAttribute() + { + assertSingleDefinitionInSameModule( + """ + DEFINE DATA + LOCAL + 1 #CVAR (C) + END-DEFINE + + WRITE #HI(CV=#C${}$VAR) + END + """, + 2, 2 + ); + } + @Test void definitionsShouldReturnTheDefinitionOfQualifiedVariablesInAssignments() { diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index 64c19d2c7..c774f365f 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -874,6 +874,12 @@ private void consumeIdentifierOrKeyword() return; } + if (inParens && scanner.peekText("CV=")) + { + controlVariableAttribute(); + return; + } + if (inParens && scanner.peekText("DF=")) { dateFormatAttribute(); @@ -1032,6 +1038,14 @@ && isValidIdentifierCharacter(scanner.peek())) } } + private void controlVariableAttribute() + { + scanner.start(); + scanner.advance(3); // CV= + // we intentionally don't consume more, because the variable should be IDENTIFIER + createAndAdd(SyntaxKind.CV); + } + private void editorMask() { scanner.start(); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index e7e366589..a593418f6 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -762,6 +762,6 @@ public boolean canBeIdentifier() public boolean isAttribute() { - return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS; + return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV; } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index 1d3855829..be687ebc9 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -861,7 +861,14 @@ protected void consumeAttributeDefinition(BaseSyntaxNode node) throws ParseError consumeMandatory(node, SyntaxKind.LPAREN); while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) { - consume(node); + if (consumeOptionally(node, SyntaxKind.CV)) + { + consumeVariableReferenceNode(node); + } + else + { + consume(node); + } } consumeMandatory(node, SyntaxKind.RPAREN); } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java index 8fa3687c0..97b94af54 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java @@ -161,4 +161,16 @@ void consumeIS() token(SyntaxKind.RPAREN) ); } + + @Test + void consumeCV() + { + assertTokens( + "(CV=#VAR)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.CV, "CV="), + token(SyntaxKind.IDENTIFIER, "#VAR"), + token(SyntaxKind.RPAREN) + ); + } } From c66f2b06b11c518e42ffee01ce01d1ebbdd28e11 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 12:01:35 +0200 Subject: [PATCH 15/23] Print lines of checked code --- .../org/amshove/natlint/cli/CliAnalyzer.java | 35 +++++++++++++++---- .../amshove/natparse/lexing/TokenList.java | 9 ++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java index c238b634b..7b4169efc 100644 --- a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java +++ b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; public class CliAnalyzer @@ -93,6 +94,7 @@ public int run() private final AtomicInteger filesChecked = new AtomicInteger(); private final AtomicInteger totalDiagnostics = new AtomicInteger(); private final AtomicInteger exceptions = new AtomicInteger(); + private final AtomicLong linesOfCode = new AtomicLong(); private int analyze(Path projectFilePath) { @@ -174,13 +176,17 @@ private int analyze(Path projectFilePath) System.out.println(); System.out.println("Done."); - System.out.println("Index time: " + indexTime + " ms"); - System.out.println("Check time: " + checkTime + " ms"); - System.out.println("Miss time : " + missTime + " ms"); - System.out.println("Total: " + totalTime + " ms (" + (totalTime / 1000) + "s)"); - System.out.println("Files checked: " + filesChecked.get()); - System.out.println("Total diagnostics: " + totalDiagnostics.get()); + System.out.printf("Index time: %d ms%n", indexTime); + System.out.printf("Check time: %d ms%n", checkTime); + System.out.printf("Miss time : %d ms%n", missTime); + System.out.printf("Total: %d ms (%ds)%n", totalTime, totalTime / 1000); + System.out.println(); + System.out.printf("Files checked: %,d%n", filesChecked.get()); + System.out.printf("Lines of code: %,d%n", linesOfCode.get()); + System.out.println(); + System.out.printf("Total diagnostics: %,d%n", totalDiagnostics.get()); System.out.println("Exceptions: " + exceptions.get()); + System.out.println(); System.out.println("Slowest lexed module: " + slowestLexedModule); System.out.println("Slowest parsed module: " + slowestParsedModule); System.out.println("Slowest linted module: " + (disableLinting ? "disabled" : slowestLintedModule)); @@ -203,6 +209,7 @@ private TokenList lex(NaturalFile file, ArrayList allDiagnosticsInF var lexStart = System.currentTimeMillis(); var tokens = lexer.lex(filesystem.readFile(file.getPath()), file.getPath()); var lexEnd = System.currentTimeMillis(); + countLinesOfCode(tokens); if (slowestLexedModule.milliseconds < lexEnd - lexStart) { slowestLexedModule = new SlowestModule(lexEnd - lexStart, file.getProjectRelativePath().toString()); @@ -221,6 +228,22 @@ private TokenList lex(NaturalFile file, ArrayList allDiagnosticsInF } } + private void countLinesOfCode(TokenList tokens) + { + var previousLine = -1; + var totalLines = 0; + for (var token : tokens) + { + if (token.line() != previousLine) + { + totalLines++; + previousLine = token.line(); + } + } + + linesOfCode.addAndGet(totalLines); + } + private INaturalModule parse(NaturalFile file, TokenList tokens, ArrayList allDiagnosticsInFile) { try diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java index 15c21aa14..001c69cf8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java @@ -5,10 +5,11 @@ import org.amshove.natparse.natural.project.NaturalHeader; import java.nio.file.Path; +import java.util.Iterator; import java.util.List; import java.util.stream.Stream; -public class TokenList +public class TokenList implements Iterable { public static TokenList fromTokens(Path filePath, List tokenList) { @@ -257,4 +258,10 @@ public void rollback() { currentOffset = 0; } + + @Override + public Iterator iterator() + { + return tokens.iterator(); + } } From aa98cc934c46b35baf9813500defa964ec91e8a7 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 15:33:22 +0200 Subject: [PATCH 16/23] Do not mistake attribute as system function parameter --- .../src/main/java/org/amshove/natlint/cli/CliAnalyzer.java | 4 +++- .../java/org/amshove/natparse/parsing/AbstractParser.java | 2 +- .../natparse/parsing/StatementListParserShould.java | 7 +++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java index 7b4169efc..978caaca2 100644 --- a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java +++ b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java @@ -174,15 +174,17 @@ private int analyze(Path projectFilePath) var missTime = missingEndTime - missingStartTime; var totalTime = indexTime + checkTime + missTime; + var totalTimeSeconds = totalTime / 1000; System.out.println(); System.out.println("Done."); System.out.printf("Index time: %d ms%n", indexTime); System.out.printf("Check time: %d ms%n", checkTime); System.out.printf("Miss time : %d ms%n", missTime); - System.out.printf("Total: %d ms (%ds)%n", totalTime, totalTime / 1000); + System.out.printf("Total: %d ms (%ds)%n", totalTime, totalTimeSeconds); System.out.println(); System.out.printf("Files checked: %,d%n", filesChecked.get()); System.out.printf("Lines of code: %,d%n", linesOfCode.get()); + System.out.printf("LoC/s: %,d%n", totalTimeSeconds > 0 ? (linesOfCode.get() / totalTimeSeconds) : linesOfCode.get()); System.out.println(); System.out.printf("Total diagnostics: %,d%n", totalDiagnostics.get()); System.out.println("Exceptions: " + exceptions.get()); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index be687ebc9..527864d21 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -470,7 +470,7 @@ protected IOperandNode consumeOperandNode(BaseSyntaxNode node) throws ParseError } if (peek().kind().isSystemVariable() && peek().kind().isSystemFunction()) // can be both, like *COUNTER { - return peekKind(1, SyntaxKind.LPAREN) ? consumeSystemFunctionNode(node) : consumeSystemVariableNode(node); + return peekKind(1, SyntaxKind.LPAREN) && !getKind(2).isAttribute() ? consumeSystemFunctionNode(node) : consumeSystemVariableNode(node); } if (peek().kind().isSystemVariable()) { diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index fa3e1ff64..5a8e97052 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1014,6 +1014,13 @@ void parseWriteWithAttributeDefinition() assertThat(write.descendants()).hasSize(6); } + @Test + void notParseAttributeAsIsnParameter() + { + var write = assertParsesSingleStatement("WRITE *ISN(NL=8)", IWriteNode.class); + assertThat(write.descendants()).anyMatch(n -> n instanceof ITokenNode tNode && tNode.token().kind() == SyntaxKind.NL); + } + @Test void parseWriteWithNoTitleAndNoHdr() { From 6035b081377cae9e8d30d83fe2329c1592a1cfef Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 15:54:07 +0200 Subject: [PATCH 17/23] Parse ZP --- .../org/amshove/natparse/lexing/Lexer.java | 92 ++++++++----------- .../amshove/natparse/lexing/SyntaxKind.java | 2 +- .../lexing/text/SourceTextScanner.java | 15 +++ .../LexerForAttributeControlsShould.java | 11 +++ 4 files changed, 63 insertions(+), 57 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index c774f365f..5331bbfde 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -838,64 +838,32 @@ private boolean isValidIdentifierCharacter(char character) private void consumeIdentifierOrKeyword() { - if (inParens && scanner.peekText("EM=")) + if (inParens) { - editorMask(); - return; - } - - if (inParens && scanner.peekText("AD=")) - { - attributeDefinition(); - return; - } - - if (inParens && scanner.peekText("CD=")) - { - colorDefinition(); - return; - } - - if (inParens && scanner.peekText("DY=")) - { - dynamicAttribute(); - return; - } - - if (inParens && scanner.peekText("NL=")) - { - numericLengthAttribute(); - return; - } - - if (inParens && scanner.peekText("AL=")) - { - alphanumericLengthAttribute(); - return; - } - - if (inParens && scanner.peekText("CV=")) - { - controlVariableAttribute(); - return; - } - - if (inParens && scanner.peekText("DF=")) - { - dateFormatAttribute(); - return; - } - - if (inParens && scanner.peekText("IP=")) - { - inputPromptAttribute(); - return; - } + var attributeLookahead = scanner.peekText(3); + if (attributeLookahead.charAt(attributeLookahead.length() - 1) == '=') + { + var previous = previous(); + switch (attributeLookahead) + { + case "AD=" -> attributeDefinition(); + case "AL=" -> alphanumericLengthAttribute(); + case "CD=" -> colorDefinition(); + case "CV=" -> controlVariableAttribute(); + case "DF=" -> dateFormatAttribute(); + case "DY=" -> dynamicAttribute(); + case "EM=" -> editorMask(); + case "IP=" -> inputPromptAttribute(); + case "IS=" -> identicalSuppressAttribute(); + case "NL=" -> numericLengthAttribute(); + case "ZP=" -> zeroPrintingAttribute(); + } - if (inParens && scanner.peekText("IS=")) - { - identicalSuppressAttribute(); - return; + if (previous() != previous) // check that we consumed something + { + return; + } + } } if (inParens && tokens.size() > 2) @@ -1101,6 +1069,18 @@ private void alphanumericLengthAttribute() createAndAdd(SyntaxKind.AL); } + private void zeroPrintingAttribute() + { + scanner.start(); + scanner.advance(3); // ZP= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.ZP); + } + private void dateFormatAttribute() { scanner.start(); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index a593418f6..61a5a5676 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -762,6 +762,6 @@ public boolean canBeIdentifier() public boolean isAttribute() { - return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV; + return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV || this == ZP; } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java index e4e5ca2c1..048417bec 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java @@ -170,4 +170,19 @@ public boolean peekText(String text) return true; } + + public String peekText(int length) + { + if (willPassEnd(length)) + { + return null; + } + + var builder = new StringBuilder(length); + for (var i = currentOffset; i < currentOffset + length; i++) + { + builder.append(source[i]); + } + return builder.toString(); + } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java index 97b94af54..df645ce3f 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java @@ -162,6 +162,17 @@ void consumeIS() ); } + @Test + void consumeZP() + { + assertTokens( + "(ZP=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.ZP, "ZP=OFF"), + token(SyntaxKind.RPAREN) + ); + } + @Test void consumeCV() { From 4a8ca91d9c59d358e89164d2e24e64006052593e Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 15:57:49 +0200 Subject: [PATCH 18/23] Eliminate check --- .../org/amshove/natparse/lexing/Lexer.java | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index 5331bbfde..cf5ee75ee 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -838,31 +838,28 @@ private boolean isValidIdentifierCharacter(char character) private void consumeIdentifierOrKeyword() { - if (inParens) + if (inParens && scanner.peek(2) == '=') { var attributeLookahead = scanner.peekText(3); - if (attributeLookahead.charAt(attributeLookahead.length() - 1) == '=') + var previous = previous(); + switch (attributeLookahead) { - var previous = previous(); - switch (attributeLookahead) - { - case "AD=" -> attributeDefinition(); - case "AL=" -> alphanumericLengthAttribute(); - case "CD=" -> colorDefinition(); - case "CV=" -> controlVariableAttribute(); - case "DF=" -> dateFormatAttribute(); - case "DY=" -> dynamicAttribute(); - case "EM=" -> editorMask(); - case "IP=" -> inputPromptAttribute(); - case "IS=" -> identicalSuppressAttribute(); - case "NL=" -> numericLengthAttribute(); - case "ZP=" -> zeroPrintingAttribute(); - } + case "AD=" -> attributeDefinition(); + case "AL=" -> alphanumericLengthAttribute(); + case "CD=" -> colorDefinition(); + case "CV=" -> controlVariableAttribute(); + case "DF=" -> dateFormatAttribute(); + case "DY=" -> dynamicAttribute(); + case "EM=" -> editorMask(); + case "IP=" -> inputPromptAttribute(); + case "IS=" -> identicalSuppressAttribute(); + case "NL=" -> numericLengthAttribute(); + case "ZP=" -> zeroPrintingAttribute(); + } - if (previous() != previous) // check that we consumed something - { - return; - } + if (previous() != previous) // check that we consumed something + { + return; } } From ba676c4042babd6146ef16b70e136e88ffcc317c Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 16:01:18 +0200 Subject: [PATCH 19/23] Eliminate allocation --- .../amshove/natparse/lexing/text/SourceTextScanner.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java index 048417bec..ffd238c20 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java @@ -178,11 +178,6 @@ public String peekText(int length) return null; } - var builder = new StringBuilder(length); - for (var i = currentOffset; i < currentOffset + length; i++) - { - builder.append(source[i]); - } - return builder.toString(); + return new String(source, currentOffset, length); } } From 81b90b11ff549f0cb64bb4983d5b310a594fc5a3 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 16:07:41 +0200 Subject: [PATCH 20/23] Parse SG --- .../java/org/amshove/natparse/lexing/Lexer.java | 13 +++++++++++++ .../org/amshove/natparse/lexing/SyntaxKind.java | 2 +- .../lexing/LexerForAttributeControlsShould.java | 11 +++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index cf5ee75ee..2f3ea5595 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -854,6 +854,7 @@ private void consumeIdentifierOrKeyword() case "IP=" -> inputPromptAttribute(); case "IS=" -> identicalSuppressAttribute(); case "NL=" -> numericLengthAttribute(); + case "SG=" -> signPosition(); case "ZP=" -> zeroPrintingAttribute(); } @@ -1126,6 +1127,18 @@ private void numericLengthAttribute() createAndAdd(SyntaxKind.NL); } + private void signPosition() + { + scanner.start(); + scanner.advance(3); // SG= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.SG); + } + private void dynamicAttribute() { scanner.start(); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index 61a5a5676..8ae2aaf2a 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -762,6 +762,6 @@ public boolean canBeIdentifier() public boolean isAttribute() { - return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV || this == ZP; + return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV || this == ZP || this == SG; } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java index df645ce3f..982af9101 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java @@ -173,6 +173,17 @@ void consumeZP() ); } + @Test + void consumeSP() + { + assertTokens( + "(SG=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.SG, "SG=OFF"), + token(SyntaxKind.RPAREN) + ); + } + @Test void consumeCV() { From dcf30df9cc55ad8d3bc7aa3b9ea4351d647a1cfc Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 16:26:54 +0200 Subject: [PATCH 21/23] Parse WRITE PC --- .../natparse/natural/IWritePcNode.java | 10 +++++ .../natparse/parsing/StatementListParser.java | 26 +++++++++++ .../amshove/natparse/parsing/WritePcNode.java | 45 +++++++++++++++++++ .../parsing/StatementListParserShould.java | 28 ++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java create mode 100644 libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java new file mode 100644 index 000000000..51e611d75 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java @@ -0,0 +1,10 @@ +package org.amshove.natparse.natural; + +public interface IWritePcNode extends IStatementNode +{ + ILiteralNode number(); + + boolean isVariable(); + + IOperandNode operand(); +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index 045a73760..021c2bca4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -182,6 +182,11 @@ private StatementListNode statementList(SyntaxKind endTokenKind) statementList.addStatement(writeWork()); break; } + if (peekKind(1, SyntaxKind.PC)) + { + statementList.addStatement(writeDownloadPc()); + break; + } statementList.addStatement(write()); break; case DISPLAY: @@ -446,6 +451,27 @@ private StatementNode writeWork() throws ParseError return writeWork; } + private StatementNode writeDownloadPc() throws ParseError + { + var writePc = new WritePcNode(); + consumeAnyMandatory(writePc, List.of(SyntaxKind.WRITE, SyntaxKind.DOWNLOAD)); + consumeMandatory(writePc, SyntaxKind.PC); + consumeOptionally(writePc, SyntaxKind.FILE); + writePc.setNumber(consumeLiteralNode(writePc, SyntaxKind.NUMBER_LITERAL)); + if (consumeOptionally(writePc, SyntaxKind.COMMAND)) + { + writePc.setOperand(consumeOperandNode(writePc)); + consumeAnyOptionally(writePc, List.of(SyntaxKind.SYNC, SyntaxKind.ASYNC)); + } + else + { + writePc.setVariable(consumeOptionally(writePc, SyntaxKind.VARIABLE)); + writePc.setOperand(consumeOperandNode(writePc)); + } + + return writePc; + } + private StatementNode closePc() throws ParseError { var closePc = new ClosePcNode(); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java new file mode 100644 index 000000000..906c048c4 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java @@ -0,0 +1,45 @@ +package org.amshove.natparse.parsing; + +import org.amshove.natparse.natural.ILiteralNode; +import org.amshove.natparse.natural.IOperandNode; +import org.amshove.natparse.natural.IWritePcNode; + +class WritePcNode extends StatementNode implements IWritePcNode +{ + private IOperandNode operand; + private ILiteralNode number; + private boolean isVariable; + + @Override + public ILiteralNode number() + { + return number; + } + + @Override + public boolean isVariable() + { + return isVariable; + } + + @Override + public IOperandNode operand() + { + return operand; + } + + void setNumber(ILiteralNode literalNode) + { + number = literalNode; + } + + void setOperand(IOperandNode operand) + { + this.operand = operand; + } + + void setVariable(boolean isVariable) + { + this.isVariable = isVariable; + } +} diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index 5a8e97052..40dd5c2df 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1927,4 +1927,32 @@ DEFINE PROTOTYPE VARIABLE HI RETURNS (L) assertThat(prototype.isVariable()).isTrue(); assertThat(prototype.variableReference()).isNotNull(); } + + @Test + void parseWritePcWithVariable() + { + var write = assertParsesSingleStatement("WRITE PC FILE 1 VARIABLE 'Hi'", IWritePcNode.class); + assertThat(write.isVariable()).isTrue(); + assertLiteral(write.number(), SyntaxKind.NUMBER_LITERAL); + } + + @Test + void parseWritePcWithoutVariable() + { + var write = assertParsesSingleStatement("WRITE PC FILE 1 'Hi'", IWritePcNode.class); + assertThat(write.isVariable()).isFalse(); + assertLiteral(write.operand(), SyntaxKind.STRING_LITERAL); + } + + @Test + void parseWritePcCommandSync() + { + assertParsesSingleStatement("WRITE PC 5 COMMAND 'Hi' SYNC", IWritePcNode.class); + } + + @Test + void parseWritePcCommandAsync() + { + assertParsesSingleStatement("WRITE PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); + } } From 7b813941fbc1bdaa6f3ab69458135bac41d20a06 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 16:31:04 +0200 Subject: [PATCH 22/23] Parse DOWNLOAD PC --- docs/implemented-statements.md | 2 +- .../natparse/parsing/StatementListParser.java | 3 ++ .../parsing/StatementListParserShould.java | 28 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/implemented-statements.md b/docs/implemented-statements.md index d3514d613..b1496ee85 100644 --- a/docs/implemented-statements.md +++ b/docs/implemented-statements.md @@ -49,7 +49,7 @@ partial - partially implemented to prevent false positives (7) | DELETE (SQL) | :x: | | DISPLAY | :x: | | DIVIDE | :white_check_mark: | -| DOWNLOAD PC FILE | :x: | +| DOWNLOAD PC FILE | :white_check_mark: | | EJECT | :white_check_mark: | | END | :white_check_mark: | | END TRANSACTION | :x: | diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index 021c2bca4..2de1e15e4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -125,6 +125,9 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case COMPUTE: statementList.addStatements(assignOrCompute(SyntaxKind.COMPUTE)); break; + case DOWNLOAD: + statementList.addStatement(writeDownloadPc()); + break; case REDUCE: statementList.addStatement(reduce()); break; diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index 40dd5c2df..d40c9883f 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1955,4 +1955,32 @@ void parseWritePcCommandAsync() { assertParsesSingleStatement("WRITE PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); } + + @Test + void parseDownloadPcWithVariable() + { + var download = assertParsesSingleStatement("DOWNLOAD PC FILE 1 VARIABLE 'Hi'", IWritePcNode.class); + assertThat(download.isVariable()).isTrue(); + assertLiteral(download.number(), SyntaxKind.NUMBER_LITERAL); + } + + @Test + void parseDownloadPcWithoutVariable() + { + var download = assertParsesSingleStatement("DOWNLOAD PC FILE 1 'Hi'", IWritePcNode.class); + assertThat(download.isVariable()).isFalse(); + assertLiteral(download.operand(), SyntaxKind.STRING_LITERAL); + } + + @Test + void parseDownloadPcCommandSync() + { + assertParsesSingleStatement("DOWNLOAD PC 5 COMMAND 'Hi' SYNC", IWritePcNode.class); + } + + @Test + void parseDownloadPcCommandAsync() + { + assertParsesSingleStatement("DOWNLOAD PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); + } } From 52f223115d1d9a07de44b00dc52bdd8efeebac96 Mon Sep 17 00:00:00 2001 From: Markus Amshove Date: Mon, 1 May 2023 16:44:08 +0200 Subject: [PATCH 23/23] Allow identifiers to have a LABEL reference again --- .../org/amshove/natparse/parsing/AbstractParser.java | 9 +++++++++ .../natparse/parsing/StatementListParserShould.java | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index 527864d21..4781ee741 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -781,6 +781,15 @@ protected IVariableReferenceNode consumeVariableReferenceNode(BaseSyntaxNode nod consumeMandatory(reference, SyntaxKind.RPAREN); } + if (peekKind(SyntaxKind.LPAREN) && peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) + { + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + { + consume(reference); + } + consumeMandatory(reference, SyntaxKind.RPAREN); + } + unresolvedReferences.add(reference); return reference; } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index d40c9883f..de1f82981 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1983,4 +1983,11 @@ void parseDownloadPcCommandAsync() { assertParsesSingleStatement("DOWNLOAD PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); } + + @Test + void allowLabelIdentifierAsVariableOperand() + { + var assignment = assertParsesSingleStatement("#VAR(R1.) := 5", IAssignmentStatementNode.class); + assertIsVariableReference(assignment.target(), "#VAR"); + } }