diff --git a/docs/implemented-statements.md b/docs/implemented-statements.md index ad591971d..b1496ee85 100644 --- a/docs/implemented-statements.md +++ b/docs/implemented-statements.md @@ -4,11 +4,11 @@ This document tracks the implementation status of Natural statements. Legend: -:x: - not implemented (65) +:x: - not implemented (64) :white_check_mark: - implemented or reporting (51) -partial - partially implemented to prevent false positives (6) +partial - partially implemented to prevent false positives (7) | Statement | Status | | --- |--------------------| @@ -41,7 +41,7 @@ partial - partially implemented to prevent false positives (6) | DEFINE DATA | :white_check_mark: | | DEFINE FUNCTION | partial | | DEFINE PRINTER | :white_check_mark: | -| DEFINE PROTOTYPE | :x: | +| DEFINE PROTOTYPE | partial | | DEFINE SUBROUTINE | :white_check_mark: | | DEFINE WINDOW | partial | | DEFINE WORK FILE | :white_check_mark: | @@ -49,13 +49,13 @@ partial - partially implemented to prevent false positives (6) | DELETE (SQL) | :x: | | DISPLAY | :x: | | DIVIDE | :white_check_mark: | -| DOWNLOAD PC FILE | :x: | +| DOWNLOAD PC FILE | :white_check_mark: | | EJECT | :white_check_mark: | | END | :white_check_mark: | | END TRANSACTION | :x: | | ESCAPE | :white_check_mark: | | EXAMINE | :white_check_mark: | -| EXPAND | :white_check_mark: | +| EXPAND | :white_check_mark: | | FETCH | :white_check_mark: | | FIND | :white_check_mark: | | FOR | :white_check_mark: | diff --git a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java index c238b634b..978caaca2 100644 --- a/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java +++ b/libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; public class CliAnalyzer @@ -93,6 +94,7 @@ public int run() private final AtomicInteger filesChecked = new AtomicInteger(); private final AtomicInteger totalDiagnostics = new AtomicInteger(); private final AtomicInteger exceptions = new AtomicInteger(); + private final AtomicLong linesOfCode = new AtomicLong(); private int analyze(Path projectFilePath) { @@ -172,15 +174,21 @@ private int analyze(Path projectFilePath) var missTime = missingEndTime - missingStartTime; var totalTime = indexTime + checkTime + missTime; + var totalTimeSeconds = totalTime / 1000; System.out.println(); System.out.println("Done."); - System.out.println("Index time: " + indexTime + " ms"); - System.out.println("Check time: " + checkTime + " ms"); - System.out.println("Miss time : " + missTime + " ms"); - System.out.println("Total: " + totalTime + " ms (" + (totalTime / 1000) + "s)"); - System.out.println("Files checked: " + filesChecked.get()); - System.out.println("Total diagnostics: " + totalDiagnostics.get()); + System.out.printf("Index time: %d ms%n", indexTime); + System.out.printf("Check time: %d ms%n", checkTime); + System.out.printf("Miss time : %d ms%n", missTime); + System.out.printf("Total: %d ms (%ds)%n", totalTime, totalTimeSeconds); + System.out.println(); + System.out.printf("Files checked: %,d%n", filesChecked.get()); + System.out.printf("Lines of code: %,d%n", linesOfCode.get()); + System.out.printf("LoC/s: %,d%n", totalTimeSeconds > 0 ? (linesOfCode.get() / totalTimeSeconds) : linesOfCode.get()); + System.out.println(); + System.out.printf("Total diagnostics: %,d%n", totalDiagnostics.get()); System.out.println("Exceptions: " + exceptions.get()); + System.out.println(); System.out.println("Slowest lexed module: " + slowestLexedModule); System.out.println("Slowest parsed module: " + slowestParsedModule); System.out.println("Slowest linted module: " + (disableLinting ? "disabled" : slowestLintedModule)); @@ -203,6 +211,7 @@ private TokenList lex(NaturalFile file, ArrayList allDiagnosticsInF var lexStart = System.currentTimeMillis(); var tokens = lexer.lex(filesystem.readFile(file.getPath()), file.getPath()); var lexEnd = System.currentTimeMillis(); + countLinesOfCode(tokens); if (slowestLexedModule.milliseconds < lexEnd - lexStart) { slowestLexedModule = new SlowestModule(lexEnd - lexStart, file.getProjectRelativePath().toString()); @@ -221,6 +230,22 @@ private TokenList lex(NaturalFile file, ArrayList allDiagnosticsInF } } + private void countLinesOfCode(TokenList tokens) + { + var previousLine = -1; + var totalLines = 0; + for (var token : tokens) + { + if (token.line() != previousLine) + { + totalLines++; + previousLine = token.line(); + } + } + + linesOfCode.addAndGet(totalLines); + } + private INaturalModule parse(NaturalFile file, TokenList tokens, ArrayList allDiagnosticsInFile) { try diff --git a/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java b/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java index f0636e2d8..6217e5640 100644 --- a/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java +++ b/libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java @@ -202,6 +202,23 @@ void definitionsShouldReturnTheDefinitionOfVariablesInAssignments() ); } + @Test + void definitionsShouldReturnTheDefinitionOfVariablesInCVAttribute() + { + assertSingleDefinitionInSameModule( + """ + DEFINE DATA + LOCAL + 1 #CVAR (C) + END-DEFINE + + WRITE #HI(CV=#C${}$VAR) + END + """, + 2, 2 + ); + } + @Test void definitionsShouldReturnTheDefinitionOfQualifiedVariablesInAssignments() { diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java index 33a8dc210..2f3ea5595 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java @@ -33,6 +33,7 @@ private enum LexerMode { DEFAULT, IN_DEFINE_DATA, + IN_DATA_TYPE } private LexerMode lexerMode = LexerMode.DEFAULT; @@ -55,7 +56,7 @@ public TokenList lex(String source, Path filePath) continue; } - if (consumeComment()) + if (lexerMode != LexerMode.IN_DATA_TYPE && consumeComment()) { continue; } @@ -73,10 +74,18 @@ public TokenList lex(String source, Path filePath) case '(': inParens = true; lastBeforeOpenParens = previous(); + if (lexerMode == LexerMode.IN_DEFINE_DATA && previous().kind() != SyntaxKind.LESSER_SIGN) + { + lexerMode = LexerMode.IN_DATA_TYPE; + } createAndAddCurrentSingleToken(SyntaxKind.LPAREN); continue; case ')': inParens = false; + if (lexerMode == LexerMode.IN_DATA_TYPE) + { + lexerMode = LexerMode.IN_DEFINE_DATA; + } lastBeforeOpenParens = null; createAndAddCurrentSingleToken(SyntaxKind.RPAREN); continue; @@ -767,7 +776,7 @@ private void consumeIdentifier() if (scanner.peek() == '/' && scanner.peek(1) == '*') { // Slash is a valid character for identifiers, but an asterisk is not. - // If a variable is named #MYVAR/* we can safely assume its a variable followed + // If a variable is named #MYVAR/* we can safely assume it's a variable followed // by a comment. break; } @@ -829,28 +838,30 @@ private boolean isValidIdentifierCharacter(char character) private void consumeIdentifierOrKeyword() { - if (inParens && scanner.peekText("EM=")) - { - editorMask(); - return; - } - - if (inParens && scanner.peekText("AD=")) - { - attributeDefinition(); - return; - } - - if (inParens && scanner.peekText("CD=")) + if (inParens && scanner.peek(2) == '=') { - colorDefinition(); - return; - } + var attributeLookahead = scanner.peekText(3); + var previous = previous(); + switch (attributeLookahead) + { + case "AD=" -> attributeDefinition(); + case "AL=" -> alphanumericLengthAttribute(); + case "CD=" -> colorDefinition(); + case "CV=" -> controlVariableAttribute(); + case "DF=" -> dateFormatAttribute(); + case "DY=" -> dynamicAttribute(); + case "EM=" -> editorMask(); + case "IP=" -> inputPromptAttribute(); + case "IS=" -> identicalSuppressAttribute(); + case "NL=" -> numericLengthAttribute(); + case "SG=" -> signPosition(); + case "ZP=" -> zeroPrintingAttribute(); + } - if (inParens && scanner.peekText("DY=")) - { - dynamicAttribte(); - return; + if (previous() != previous) // check that we consumed something + { + return; + } } if (inParens && tokens.size() > 2) @@ -900,6 +911,12 @@ private void consumeIdentifierOrKeyword() break; } + if (scanner.peek() == '/' && lexerMode == LexerMode.IN_DATA_TYPE) + { + // Slash is a valid character for identifiers, if we're lexing a datatype we can be pretty confident about the slash being for array dimensions + break; + } + if (scanner.peek() == '/') { // TODO: this does not work for "KEYWORD/*", eg. END-SUBROUTINE/* bla bla kindHint = SyntaxKind.IDENTIFIER; @@ -987,6 +1004,14 @@ && isValidIdentifierCharacter(scanner.peek())) } } + private void controlVariableAttribute() + { + scanner.start(); + scanner.advance(3); // CV= + // we intentionally don't consume more, because the variable should be IDENTIFIER + createAndAdd(SyntaxKind.CV); + } + private void editorMask() { scanner.start(); @@ -1030,7 +1055,91 @@ private void attributeDefinition() createAndAdd(SyntaxKind.AD); } - private void dynamicAttribte() + private void alphanumericLengthAttribute() + { + scanner.start(); + scanner.advance(3); // AL= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.AL); + } + + private void zeroPrintingAttribute() + { + scanner.start(); + scanner.advance(3); // ZP= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.ZP); + } + + private void dateFormatAttribute() + { + scanner.start(); + scanner.advance(3); // DF= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.DF); + } + + private void inputPromptAttribute() + { + scanner.start(); + scanner.advance(3); // IP= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.IP); + } + + private void identicalSuppressAttribute() + { + scanner.start(); + scanner.advance(3); // IS= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.IS); + } + + private void numericLengthAttribute() + { + scanner.start(); + scanner.advance(3); // NL= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.NL); + } + + private void signPosition() + { + scanner.start(); + scanner.advance(3); // SG= + while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')') + { + scanner.advance(); + } + + createAndAdd(SyntaxKind.SG); + } + + private void dynamicAttribute() { scanner.start(); scanner.advance(3); // DY= diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java index 10831f51e..8ae2aaf2a 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java @@ -2,6 +2,7 @@ public enum SyntaxKind { + EOF(false, false, false), // end of file LBRACKET(false, false, false), RBRACKET(false, false, false), LPAREN(false, false, false), @@ -107,8 +108,8 @@ public enum SyntaxKind PROGRAM(true, true, false), ETID(false, true, false), CPU_TIME(false, true, false), - LBOUND(false, true, false), - UBOUND(false, true, false), + LBOUND(false, false, true), + UBOUND(false, false, true), SERVER_TYPE(false, true, false), // Kcheck reserved keywords @@ -758,4 +759,9 @@ public boolean canBeIdentifier() { return canBeIdentifier; } + + public boolean isAttribute() + { + return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV || this == ZP || this == SG; + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java index 15c21aa14..001c69cf8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/TokenList.java @@ -5,10 +5,11 @@ import org.amshove.natparse.natural.project.NaturalHeader; import java.nio.file.Path; +import java.util.Iterator; import java.util.List; import java.util.stream.Stream; -public class TokenList +public class TokenList implements Iterable { public static TokenList fromTokens(Path filePath, List tokenList) { @@ -257,4 +258,10 @@ public void rollback() { currentOffset = 0; } + + @Override + public Iterator iterator() + { + return tokens.iterator(); + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java index e4e5ca2c1..ffd238c20 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/lexing/text/SourceTextScanner.java @@ -170,4 +170,14 @@ public boolean peekText(String text) return true; } + + public String peekText(int length) + { + if (willPassEnd(length)) + { + return null; + } + + return new String(source, currentOffset, length); + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java new file mode 100644 index 000000000..203c95c76 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IDefinePrototypeNode.java @@ -0,0 +1,22 @@ +package org.amshove.natparse.natural; + +import org.amshove.natparse.lexing.SyntaxToken; + +import javax.annotation.Nullable; + +public interface IDefinePrototypeNode extends IStatementNode +{ + /** + * Contains the name of the prototype. Could be a reference to an external function or a variable reference if + * {@code isVariable()} is true. + */ + SyntaxToken nameToken(); + + boolean isVariable(); + + /** + * The reference to the variable containing the function name. Is filled if {@code isVariable()} is true. + */ + @Nullable + IVariableReferenceNode variableReference(); +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java index 1517cbd28..bad01c0a8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IVariableNode.java @@ -17,6 +17,8 @@ public non-sealed interface IVariableNode extends IReferencableNode, IParameterD ReadOnlyList dimensions(); + boolean isInView(); + default boolean isArray() { return dimensions() != null && dimensions().size() > 0; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java new file mode 100644 index 000000000..51e611d75 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/IWritePcNode.java @@ -0,0 +1,10 @@ +package org.amshove.natparse.natural; + +public interface IWritePcNode extends IStatementNode +{ + ILiteralNode number(); + + boolean isVariable(); + + IOperandNode operand(); +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java b/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java index 08ff74974..d58905a06 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/natural/builtin/BuiltInFunctionTable.java @@ -172,10 +172,10 @@ public class BuiltInFunctionTable unmodifiableVariable(SyntaxKind.INIT_PROGRAM, """ Return the name of program (transaction) currently executing as Natural. """, ALPHANUMERIC, 8), - unmodifiableVariable(SyntaxKind.LBOUND, """ + function(SyntaxKind.LBOUND, """ Returns the current lower boundary (index value) of an array for the specified dimension(s) (1, 2 or 3) or for all dimensions (asterisk (*) notation). """, INTEGER, 4), - unmodifiableVariable(SyntaxKind.UBOUND, """ + function(SyntaxKind.UBOUND, """ Returns the current upper boundary (index value) of an array for the specified dimension(s) (1, 2 or 3) or for all dimensions (asterisk (*) notation). """, INTEGER, 4), unmodifiableVariable(SyntaxKind.SERVER_TYPE, """ diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java index d86482df8..4781ee741 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java @@ -96,6 +96,19 @@ protected boolean peekKind(int offset, SyntaxKind kind) return !isAtEnd(offset) && peek(offset).kind() == kind; } + /** + * Returns the kind of the token at the given offset. If the offset is out of bounds, returns EOF. + */ + protected SyntaxKind getKind(int offset) + { + if (isAtEnd(offset)) + { + return SyntaxKind.EOF; + } + + return peek(offset).kind(); + } + protected boolean peekKind(SyntaxKind kind) { return peekKind(0, kind); @@ -327,7 +340,21 @@ protected StatementNode identifierReference() throws ParseError } var node = symbolReferenceNode(token); - return new SyntheticVariableStatementNode(node); + var variableNode = new SyntheticVariableStatementNode(node); + if (peekKind(SyntaxKind.LPAREN) + && !getKind(1).isAttribute() + && !peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) + { + consumeMandatory(node, SyntaxKind.LPAREN); + variableNode.addDimension(consumeArrayAccess(variableNode)); + while (peekKind(SyntaxKind.COMMA)) + { + consume(variableNode); + variableNode.addDimension(consumeArrayAccess(variableNode)); + } + consumeMandatory(variableNode, SyntaxKind.RPAREN); + } + return variableNode; } protected FunctionCallNode functionCall(SyntaxToken token) throws ParseError @@ -443,7 +470,7 @@ protected IOperandNode consumeOperandNode(BaseSyntaxNode node) throws ParseError } if (peek().kind().isSystemVariable() && peek().kind().isSystemFunction()) // can be both, like *COUNTER { - return peekKind(1, SyntaxKind.LPAREN) ? consumeSystemFunctionNode(node) : consumeSystemVariableNode(node); + return peekKind(1, SyntaxKind.LPAREN) && !getKind(2).isAttribute() ? consumeSystemFunctionNode(node) : consumeSystemVariableNode(node); } if (peek().kind().isSystemVariable()) { @@ -742,7 +769,7 @@ protected IVariableReferenceNode consumeVariableReferenceNode(BaseSyntaxNode nod previousNode = reference; node.addNode(reference); - if (peekKind(SyntaxKind.LPAREN) && !peekKind(1, SyntaxKind.AD)) + if (peekKind(SyntaxKind.LPAREN) && !getKind(1).isAttribute() && !peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) { consumeMandatory(reference, SyntaxKind.LPAREN); reference.addDimension(consumeArrayAccess(reference)); @@ -754,11 +781,20 @@ protected IVariableReferenceNode consumeVariableReferenceNode(BaseSyntaxNode nod consumeMandatory(reference, SyntaxKind.RPAREN); } + if (peekKind(SyntaxKind.LPAREN) && peekKind(1, SyntaxKind.LABEL_IDENTIFIER)) + { + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + { + consume(reference); + } + consumeMandatory(reference, SyntaxKind.RPAREN); + } + unresolvedReferences.add(reference); return reference; } - protected IOperandNode consumeArrayAccess(VariableReferenceNode reference) throws ParseError + protected IOperandNode consumeArrayAccess(BaseSyntaxNode reference) throws ParseError { if (peekKind(SyntaxKind.ASTERISK) && peekKind(1, SyntaxKind.RPAREN)) { @@ -832,7 +868,17 @@ protected void consumeAttributeDefinition(BaseSyntaxNode node) throws ParseError // this was built for CALLNAT, where a variable reference as parameter can have attribute definitions (only AD) // might be reusable for WRITE, DISPLAY, etc. for all kind of operands, but has to be fleshed out then consumeMandatory(node, SyntaxKind.LPAREN); - consumeMandatory(node, SyntaxKind.AD); + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + { + if (consumeOptionally(node, SyntaxKind.CV)) + { + consumeVariableReferenceNode(node); + } + else + { + consume(node); + } + } consumeMandatory(node, SyntaxKind.RPAREN); } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java index 5041fe765..3fe17140c 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java @@ -153,9 +153,35 @@ private ScopeNode scope() throws ParseError } } + passDownArrayDimensions(scopeNode); + return scopeNode; } + private void passDownArrayDimensions(ScopeNode scope) + { + for (var variable : scope.variables()) + { + inheritDimensions(variable); + } + } + + private void inheritDimensions(IVariableNode variable) + { + if (variable.parent()instanceof IVariableNode parentVariable && parentVariable.isArray()) + { + ((VariableNode) variable).inheritDimensions(parentVariable.dimensions()); + } + + if (variable instanceof GroupNode group) + { + for (var subVariable : group.variables()) + { + inheritDimensions(subVariable); + } + } + } + private UsingNode using() throws ParseError { var using = new UsingNode(); @@ -327,7 +353,10 @@ private GroupNode groupVariable(VariableNode variable) throws ParseError { for (var dimension : variable.dimensions()) { - groupNode.addDimension((ArrayDimension) dimension); + if (!groupNode.dimensions.contains(dimension)) + { + groupNode.addDimension((ArrayDimension) dimension); + } } } @@ -801,7 +830,7 @@ private void addArrayDimension(VariableNode variable) throws ParseError throw new ParseError(peek()); } - while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN) && !peekKind(SyntaxKind.COMMA)) { var dimension = new ArrayDimension(); var lowerBound = extractArrayBound(new TokenNode(peek()), dimension); @@ -827,6 +856,10 @@ private void addArrayDimension(VariableNode variable) throws ParseError dimension.setLowerBound(lowerBound); dimension.setUpperBound(upperBound); variable.addDimension(dimension); + while (!isAtEnd() && !peekKind(SyntaxKind.COMMA) && !peekKind(SyntaxKind.RPAREN)) + { + consume(dimension); + } } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java new file mode 100644 index 000000000..df1a400a2 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/DefinePrototypeNode.java @@ -0,0 +1,44 @@ +package org.amshove.natparse.parsing; + +import org.amshove.natparse.lexing.SyntaxToken; +import org.amshove.natparse.natural.IDefinePrototypeNode; +import org.amshove.natparse.natural.IOperandNode; +import org.amshove.natparse.natural.IVariableReferenceNode; + +import javax.annotation.Nullable; + +class DefinePrototypeNode extends StatementNode implements IDefinePrototypeNode +{ + + private SyntaxToken prototypeName; + private IVariableReferenceNode variableReference; + + @Override + public SyntaxToken nameToken() + { + return variableReference != null ? variableReference.referencingToken() : prototypeName; + } + + @Override + public boolean isVariable() + { + return variableReference != null; + } + + @Nullable + @Override + public IVariableReferenceNode variableReference() + { + return variableReference; + } + + void setPrototype(SyntaxToken token) + { + prototypeName = token; + } + + void setVariableReference(IVariableReferenceNode reference) + { + this.variableReference = reference; + } +} diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java index 0ffe995e5..ffb238373 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserError.java @@ -43,7 +43,8 @@ public enum ParserError INVALID_LITERAL_VALUE("NPP038"), REFERENCE_NOT_MUTABLE("NPP039"), UNSUPPORTED_PROGRAMMING_MODE("NPP040"), - INVALID_MODULE_TYPE("NPP041"); + INVALID_MODULE_TYPE("NPP041"), + INVALID_ARRAY_ACCESS("NPP042"); private final String id; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java index d65155c1f..6e739a1b8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ParserErrors.java @@ -518,4 +518,13 @@ public static IDiagnostic invalidModuleType(String message, SyntaxToken errorTok ParserError.INVALID_MODULE_TYPE ); } + + public static IDiagnostic invalidArrayAccess(SyntaxToken token, String message) + { + return ParserDiagnostic.create( + message, + token, + ParserError.INVALID_ARRAY_ACCESS + ); + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java index 19773d339..1c6e21ee4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/RedefinitionNode.java @@ -13,6 +13,10 @@ class RedefinitionNode extends GroupNode implements IRedefinitionNode public RedefinitionNode(VariableNode variable) { super(variable); + for (var inheritedDimension : variable.dimensions) + { + addDimension((ArrayDimension) inheritedDimension); + } } @Override diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java index 3444a8332..2de1e15e4 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/StatementListParser.java @@ -125,6 +125,9 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case COMPUTE: statementList.addStatements(assignOrCompute(SyntaxKind.COMPUTE)); break; + case DOWNLOAD: + statementList.addStatement(writeDownloadPc()); + break; case REDUCE: statementList.addStatement(reduce()); break; @@ -182,6 +185,11 @@ private StatementListNode statementList(SyntaxKind endTokenKind) statementList.addStatement(writeWork()); break; } + if (peekKind(1, SyntaxKind.PC)) + { + statementList.addStatement(writeDownloadPc()); + break; + } statementList.addStatement(write()); break; case DISPLAY: @@ -211,8 +219,10 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case PRINTER -> statementList.addStatement(definePrinter()); case WINDOW -> statementList.addStatement(defineWindow()); case WORK -> statementList.addStatement(defineWork()); - case PROTOTYPE, DATA -> - { // not implemented statements. DATA needs to be handled when parsing functions and external subroutines + case PROTOTYPE -> statementList.addStatement(definePrototype()); + case DATA -> + { + // can this even happen? tokens.advance(); tokens.advance(); } @@ -265,6 +275,15 @@ private StatementListNode statementList(SyntaxKind endTokenKind) case TERMINATE: statementList.addStatement(terminate()); break; + case LPAREN: + if (getKind(1).isAttribute()) + { + // Workaround for attributes. Should be added to the operand they belong to. + var tokenNode = new SyntheticTokenStatementNode(); + consumeAttributeDefinition(tokenNode); + statementList.addStatement(tokenNode); + break; + } case DECIDE: if (peekKind(1, SyntaxKind.FOR)) { @@ -342,6 +361,46 @@ private StatementListNode statementList(SyntaxKind endTokenKind) return statementList; } + private StatementNode definePrototype() throws ParseError + { + if (peekKind(2, SyntaxKind.FOR) || peekKind(2, SyntaxKind.VARIABLE)) + { + return definePrototypeVariable(); + } + + var prototype = new DefinePrototypeNode(); + var opening = consumeMandatory(prototype, SyntaxKind.DEFINE); + consumeMandatory(prototype, SyntaxKind.PROTOTYPE); + + var name = consumeMandatoryIdentifier(prototype); // TODO: Sideload + prototype.setPrototype(name); + while (!isAtEnd() && !peekKind(SyntaxKind.END_PROTOTYPE)) + { + consume(prototype); // incomplete + } + + consumeMandatoryClosing(prototype, SyntaxKind.END_PROTOTYPE, opening); + return prototype; + } + + private StatementNode definePrototypeVariable() throws ParseError + { + var prototype = new DefinePrototypeNode(); + var opening = consumeMandatory(prototype, SyntaxKind.DEFINE); + consumeMandatory(prototype, SyntaxKind.PROTOTYPE); + consumeOptionally(prototype, SyntaxKind.FOR); + consumeMandatory(prototype, SyntaxKind.VARIABLE); + + prototype.setVariableReference(consumeVariableReferenceNode(prototype)); + while (!isAtEnd() && !peekKind(SyntaxKind.END_PROTOTYPE)) + { + consume(prototype); // incomplete + } + + consumeMandatoryClosing(prototype, SyntaxKind.END_PROTOTYPE, opening); + return prototype; + } + private StatementNode terminate() throws ParseError { var terminate = new TerminateNode(); @@ -395,6 +454,27 @@ private StatementNode writeWork() throws ParseError return writeWork; } + private StatementNode writeDownloadPc() throws ParseError + { + var writePc = new WritePcNode(); + consumeAnyMandatory(writePc, List.of(SyntaxKind.WRITE, SyntaxKind.DOWNLOAD)); + consumeMandatory(writePc, SyntaxKind.PC); + consumeOptionally(writePc, SyntaxKind.FILE); + writePc.setNumber(consumeLiteralNode(writePc, SyntaxKind.NUMBER_LITERAL)); + if (consumeOptionally(writePc, SyntaxKind.COMMAND)) + { + writePc.setOperand(consumeOperandNode(writePc)); + consumeAnyOptionally(writePc, List.of(SyntaxKind.SYNC, SyntaxKind.ASYNC)); + } + else + { + writePc.setVariable(consumeOptionally(writePc, SyntaxKind.VARIABLE)); + writePc.setOperand(consumeOperandNode(writePc)); + } + + return writePc; + } + private StatementNode closePc() throws ParseError { var closePc = new ClosePcNode(); @@ -1420,6 +1500,8 @@ private StatementNode display() throws ParseError return display; } + private static final Set OPTIONAL_WRITE_FLAGS = Set.of(SyntaxKind.NOTITLE, SyntaxKind.NOHDR, SyntaxKind.USING, SyntaxKind.MAP, SyntaxKind.FORM, SyntaxKind.TITLE, SyntaxKind.LEFT, SyntaxKind.JUSTIFIED, SyntaxKind.UNDERLINED); + private StatementNode write() throws ParseError { var write = new WriteNode(); @@ -1442,10 +1524,28 @@ private StatementNode write() throws ParseError consumeMandatory(write, SyntaxKind.RPAREN); } - consumeOptionally(write, SyntaxKind.NOTITLE); - consumeOptionally(write, SyntaxKind.NOHDR); + while (consumeAnyOptionally(write, OPTIONAL_WRITE_FLAGS)) + { + // advances automatically + } + while (!isAtEnd() && !isStatementStart()) + { + if (peekKind(SyntaxKind.LPAREN) && getKind(1).isAttribute()) + { + consumeAttributeDefinition(write); + } + else + { + if ((consumeOptionally(write, SyntaxKind.NO) && consumeOptionally(write, SyntaxKind.PARAMETER)) + || !isOperand()) + { + break; + } + consumeOperandNode(write); + } + } - // TODO: Actual operands to WRITE not parsed + // TODO: Actual operands to WRITE not added as operands return write; } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java index 0042969e3..379c647cc 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SyntheticVariableStatementNode.java @@ -5,10 +5,14 @@ import org.amshove.natparse.lexing.SyntaxToken; import org.amshove.natparse.natural.*; +import java.util.ArrayList; +import java.util.List; + // TODO: Only exists until all statements are parse-able class SyntheticVariableStatementNode extends StatementNode implements IVariableReferenceNode { private final SymbolReferenceNode node; + private final List dimensions = new ArrayList<>(); public SyntheticVariableStatementNode(SymbolReferenceNode node) { @@ -66,6 +70,11 @@ public IPosition diagnosticPosition() @Override public ReadOnlyList dimensions() { - return ReadOnlyList.of(); // TODO: Do something + return ReadOnlyList.from(dimensions); + } + + void addDimension(IOperandNode dimension) + { + dimensions.add(dimension); } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java index 4d85f2984..d72c33a5e 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/SystemFunctionNode.java @@ -10,7 +10,7 @@ class SystemFunctionNode extends BaseSyntaxNode implements ISystemFunctionNode { - private List parameter = new ArrayList<>(); + private final List parameter = new ArrayList<>(); private SyntaxKind systemFunction; @Override diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java index 02072099f..806516683 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/TypeChecker.java @@ -7,6 +7,7 @@ import org.amshove.natparse.natural.builtin.BuiltInFunctionTable; import org.amshove.natparse.natural.builtin.IBuiltinFunctionDefinition; import org.amshove.natparse.natural.builtin.SystemVariableDefinition; +import org.amshove.natparse.natural.conditionals.ISpecifiedCriteriaNode; import java.util.ArrayList; import java.util.List; @@ -67,6 +68,100 @@ private void checkNode(ISyntaxNode node) { checkAlphanumericInitLength(typedVariableNode); } + + if (node instanceof IVariableReferenceNode variableReference) + { + checkVariableReference(variableReference); + } + } + + private void checkVariableReference(IVariableReferenceNode variableReference) + { + if (!(variableReference.reference()instanceof IVariableNode target)) + { + return; + } + + if (variableReference.dimensions().hasItems() && !target.isArray()) + { + if (!isPeriodicGroup(target))// periodic groups need to have their index specified. not allowed for "normal" groups + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Using index access for a reference to non-array %s".formatted(target.name()) + ) + ); + } + } + + if (variableReference.dimensions().isEmpty() && (target.isArray() || isPeriodicGroup(target))) + { + if (!doesNotNeedDimensionInParentStatement(variableReference)) + { + var message = isPeriodicGroup(target) ? "a periodic group" : "an array"; + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Missing index access, because %s is %s".formatted(target.name(), message) + ) + ); + } + } + + if (variableReference.dimensions().hasItems() && target.dimensions().hasItems() + && variableReference.dimensions().size() != target.dimensions().size()) + { + diagnostics.add( + ParserErrors.invalidArrayAccess( + variableReference.referencingToken(), + "Missing dimensions in array access. Got %d dimensions but %s has %d".formatted( + variableReference.dimensions().size(), + target.name(), + target.dimensions().size() + ) + ) + ); + } + } + + private boolean isPeriodicGroup(IVariableNode variable) + { + if (!(variable instanceof IGroupNode group)) + { + return false; + } + + if (!group.isInView()) + { + return false; + } + + var nextLevel = variable.level() + 1; + for (var periodicMember : group.variables()) + { + if (periodicMember.level() == nextLevel && !periodicMember.isArray()) + { + return false; + } + } + + return true; + } + + private boolean doesNotNeedDimensionInParentStatement(IVariableReferenceNode reference) + { + var parent = reference.parent(); + if (parent instanceof ISystemFunctionNode systemFunction) + { + var theFunction = systemFunction.systemFunction(); + return theFunction == SyntaxKind.OCC || theFunction == SyntaxKind.OCCURRENCE || theFunction == SyntaxKind.UBOUND || theFunction == SyntaxKind.LBOUND; + } + + return parent instanceof IExpandArrayNode + || parent instanceof IReduceArrayNode + || parent instanceof IResizeArrayNode + || parent instanceof ISpecifiedCriteriaNode; } private void checkWriteWork(IWriteWorkNode writeWork) diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java index f70faa7fa..1b8afe3a8 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/VariableNode.java @@ -2,12 +2,15 @@ import org.amshove.natparse.IPosition; import org.amshove.natparse.NaturalParseException; +import org.amshove.natparse.NodeUtil; import org.amshove.natparse.ReadOnlyList; import org.amshove.natparse.lexing.SyntaxToken; import org.amshove.natparse.natural.*; import javax.annotation.Nonnull; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; import java.util.List; class VariableNode extends BaseSyntaxNode implements IVariableNode @@ -118,6 +121,12 @@ public IPosition position() return declaration; } + @Override + public boolean isInView() + { + return NodeUtil.findFirstParentOfType(this, IViewNode.class) != null; + } + void setLevel(int level) { this.level = level; @@ -141,4 +150,18 @@ void addDimension(ArrayDimension dimension) dimensions.add(dimension); addNode(dimension); } + + /** + * Inherits all the given dimensions if they're not specified for this variable yet. + */ + void inheritDimensions(ReadOnlyList dimensions) + { + for (var dimension : dimensions) + { + if (!this.dimensions.contains(dimension)) + { + this.dimensions.add(0, dimension); // add inhereted dimensions first, as they're defined first + } + } + } } diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java index 396a1fd25..fcf72de7e 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewNode.java @@ -31,6 +31,12 @@ public IDataDefinitionModule ddm() return ddm; } + @Override + public boolean isInView() + { + return false; + } + void setDdm(IDataDefinitionModule ddm) { this.ddm = ddm; diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java index f6d56d812..51d061788 100644 --- a/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/ViewParser.java @@ -97,7 +97,7 @@ private VariableNode variable() throws ParseError if (peek().kind() == SyntaxKind.NUMBER_LITERAL || (peek().kind().isIdentifier() && isVariableDeclared(peek().symbolName()))) { - addArrayDimension(variable); + addArrayDimensions(variable); var typedDdmArrayVariable = typedVariableFromDdm(variable); consumeMandatory(typedDdmArrayVariable, SyntaxKind.RPAREN); return typedDdmArrayVariable; @@ -254,7 +254,7 @@ private void addArrayDimension(VariableNode variable) throws ParseError throw new ParseError(peek()); } - while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN)) + while (!isAtEnd() && !peekKind(SyntaxKind.RPAREN) && !peekKind(SyntaxKind.COMMA)) { var dimension = new ArrayDimension(); var lowerBound = extractArrayBound(new TokenNode(peek()), dimension); diff --git a/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java b/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java new file mode 100644 index 000000000..906c048c4 --- /dev/null +++ b/libs/natparse/src/main/java/org/amshove/natparse/parsing/WritePcNode.java @@ -0,0 +1,45 @@ +package org.amshove.natparse.parsing; + +import org.amshove.natparse.natural.ILiteralNode; +import org.amshove.natparse.natural.IOperandNode; +import org.amshove.natparse.natural.IWritePcNode; + +class WritePcNode extends StatementNode implements IWritePcNode +{ + private IOperandNode operand; + private ILiteralNode number; + private boolean isVariable; + + @Override + public ILiteralNode number() + { + return number; + } + + @Override + public boolean isVariable() + { + return isVariable; + } + + @Override + public IOperandNode operand() + { + return operand; + } + + void setNumber(ILiteralNode literalNode) + { + number = literalNode; + } + + void setOperand(IOperandNode operand) + { + this.operand = operand; + } + + void setVariable(boolean isVariable) + { + this.isVariable = isVariable; + } +} diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java index 0e927d3dd..982af9101 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForAttributeControlsShould.java @@ -106,4 +106,93 @@ void consumeComplexDynamicAttributes() token(SyntaxKind.RPAREN) ); } + + @Test + void consumeNL() + { + assertTokens( + "(NL=12,7)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.NL, "NL=12,7"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeAL() + { + assertTokens( + "(AL=20)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.AL, "AL=20"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeDF() + { + assertTokens( + "(DF=S)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.DF, "DF=S"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeIP() + { + assertTokens( + "(IP=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.IP, "IP=OFF"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeIS() + { + assertTokens( + "(IS=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.IS, "IS=OFF"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeZP() + { + assertTokens( + "(ZP=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.ZP, "ZP=OFF"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeSP() + { + assertTokens( + "(SG=OFF)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.SG, "SG=OFF"), + token(SyntaxKind.RPAREN) + ); + } + + @Test + void consumeCV() + { + assertTokens( + "(CV=#VAR)", + token(SyntaxKind.LPAREN), + token(SyntaxKind.CV, "CV="), + token(SyntaxKind.IDENTIFIER, "#VAR"), + token(SyntaxKind.RPAREN) + ); + } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java new file mode 100644 index 000000000..710bc7295 --- /dev/null +++ b/libs/natparse/src/test/java/org/amshove/natparse/lexing/LexerForDatatypesShould.java @@ -0,0 +1,62 @@ +package org.amshove.natparse.lexing; + +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +class LexerForDatatypesShould extends AbstractLexerTest +{ + @Test + void recognizeSimpleDataTypes() + { + assertTokens("A10", SyntaxKind.IDENTIFIER); + } + + @Test + void recognizeArrays() + { + assertTokens("A10/*", SyntaxKind.IDENTIFIER, SyntaxKind.SLASH, SyntaxKind.ASTERISK); + } + + @Test + void recognizeArraysWithWhitespace() + { + assertTokens("A10/ 1: 10", SyntaxKind.IDENTIFIER, SyntaxKind.SLASH, SyntaxKind.NUMBER_LITERAL, SyntaxKind.COLON, SyntaxKind.NUMBER_LITERAL); + } + + @Override + protected void assertTokens(String source, SyntaxKind... expectedKinds) + { + var tokens = lexSource(""" + DEFINE DATA LOCAL + 1 #VAR (%s) + END-DEFINE + """.formatted(source)); + + while (tokens.peek().kind() != SyntaxKind.LPAREN) + { + tokens.advance(); + } + tokens.advance(); + + for (var expectedKind : expectedKinds) + { + var actualToken = tokens.advance(); + assertThat(actualToken.kind()) + .as("Tokens from here to end: " + tokens.subrange(tokens.getCurrentOffset(), tokens.size() - 1).stream().map(SyntaxToken::kind).map(Enum::name).collect(Collectors.joining(", "))) + .isEqualTo(expectedKind); + } + + assertThat(tokens.peek().kind()) + .as("Expected data type end token (RPAREN)") + .isEqualTo(SyntaxKind.RPAREN); + } +} diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java index 064a272eb..64281e242 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserDiagnosticTest.java @@ -8,6 +8,6 @@ class DefineDataParserDiagnosticTest extends ResourceFolderBasedTest @TestFactory Iterable diagnosticTests() { - return testFolder("definedatadiagnostics"); + return testFolder("definedatadiagnostics", "multiDimensionalViewArray"); } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java index cb3997dbc..bdfa77d0f 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java @@ -703,6 +703,21 @@ void parseAnArrayWithLowerAndUpperBoundBeingReferences() assertThat(theGroup.dimensions().first().upperBound()).isEqualTo(9); } + @Test + void parseTheNumberOfDimensionsCorrectly() + { + var defineData = assertParsesWithoutDiagnostics(""" + define data local + 1 #GROUP (1:2,1:120) + 2 #VAR (A10) + end-define + """); + + var group = assertNodeType(defineData.variables().first(), IGroupNode.class); + assertThat(group.dimensions()).hasSize(2); + assertThat(group.variables().first().dimensions()).hasSize(2); + } + @Test void addAReferenceToTheConstantForConstArrayDimension() { @@ -797,6 +812,84 @@ void notReportALengthDiagnosticForNestedRedefineVariables() """); } + @Test + void inheritArrayDimensionsInNestedRedefines() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 01 UPPERVAR(A99) + 01 REDEFINE UPPERVAR + 02 #INNERGROUPARR(1:33) + 03 #INNERVAR(A2) + 03 REDEFINE #INNERVAR + 04 #CHAR1(A1) + 04 #CHAR2(A1) + 03 #ID(A1) + END-DEFINE + """); + + var firstRedefine = assertNodeType(defineData.variables().get(1), IRedefinitionNode.class); + var innerGroupArray = assertNodeType(firstRedefine.variables().first(), IGroupNode.class); + var firstVarInGroup = assertNodeType(innerGroupArray.variables().first(), ITypedVariableNode.class); + assertThat(firstVarInGroup.dimensions()).hasSize(1); + + var redefineOfFirstVarInGroup = assertNodeType(innerGroupArray.variables().get(1), IRedefinitionNode.class); + + assertThat(redefineOfFirstVarInGroup.variables().first().name()).isEqualTo("#CHAR1"); + assertThat(redefineOfFirstVarInGroup.variables().first().dimensions()).hasSize(1); // #CHAR1 + assertThat(redefineOfFirstVarInGroup.variables().last().name()).isEqualTo("#CHAR2"); + assertThat(redefineOfFirstVarInGroup.variables().last().dimensions()).hasSize(1); // #CHAR2 + + assertThat(innerGroupArray.variables().last().name()).isEqualTo("#ID"); + assertThat(innerGroupArray.variables().last().dimensions()).hasSize(1); + } + + @Test + void inheritArrayDimensionsInViews() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 MY-VIEW VIEW MY-DDM + 2 A-GROUP(1:12) + 3 A-VAR (N12,2) + END-DEFINE + """); + + var view = assertNodeType(defineData.variables().first(), IViewNode.class); + var group = assertNodeType(view.variables().first(), IGroupNode.class); + assertThat(group.dimensions()).hasSize(1); + var variable = assertNodeType(group.variables().first(), ITypedVariableNode.class); + assertThat(variable.dimensions()).hasSize(1); + } + + @Test + void parseTheCorrectNumberOfDimensionsForTypesWithMath() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 #VAR (N12) CONST<5> + 1 #ARR (A10/1:#VAR+1) + END-DEFINE + """); + + var array = findVariable(defineData, "#ARR", ITypedVariableNode.class); + assertThat(array.dimensions()).hasSize(1); + } + + @Test + void parseTheCorrectNumberOfDimensionsForTypesWithMathAndComma() + { + var defineData = assertParsesWithoutDiagnostics(""" + DEFINE DATA LOCAL + 1 #VAR (N12) CONST<5> + 1 #ARR (A10/1:#VAR+ 1,1 :20) + END-DEFINE + """); + + var array = findVariable(defineData, "#ARR", ITypedVariableNode.class); + assertThat(array.dimensions()).hasSize(2); + } + @Test void redefineGroups() { @@ -1043,6 +1136,21 @@ void allowVAsUnboundInParameterScope() assertThat(variable.dimensions().first().isUpperUnbound()).isTrue(); } + @Test + void parseArrayBoundsWithNastyWhitespace() + { + var defineData = assertParsesWithoutDiagnostics(""" + define data + parameter + 1 #p-array (A3/ 1: 50) + end-define + """); + + var variable = findVariable(defineData, "#p-array", ITypedVariableNode.class); + assertThat(variable.dimensions().first().lowerBound()).isEqualTo(1); + assertThat(variable.dimensions().first().upperBound()).isEqualTo(50); + } + @Test void parseArrayInitialValues() { diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java index 1694bed1a..63de9de71 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/OperandParsingTests.java @@ -468,4 +468,24 @@ void parseArrayIndicesWhichMightBeMistakenAsDecimalNumbers() var reference = assertIsVariableReference(operand, "#ARR"); assertThat(reference.dimensions()).hasSize(3); } + + @ParameterizedTest + @ValueSource(strings = + { + "DY='021I'01", "CD=RE", "AD=IO" + }) + void doNotParseAttributesAsArrayIndex(String parens) + { + var operand = parseOperand("#ARR(%s)".formatted(parens)); + var reference = assertIsVariableReference(operand, "#ARR"); + assertThat(reference.dimensions()).isEmpty(); + } + + @Test + void doNotParseLabelReferencesAsArrayIndex() + { + var operand = parseOperand("#ARR(R1.)"); + var reference = assertIsVariableReference(operand, "#ARR"); + assertThat(reference.dimensions()).isEmpty(); + } } diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java index a6d2077a7..5236d508c 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/ResourceFolderBasedTest.java @@ -14,6 +14,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static org.assertj.core.api.Fail.fail; @@ -22,6 +23,14 @@ public abstract class ResourceFolderBasedTest { protected Iterable testFolder(String relativeFolderPath) + { + return testFolder(relativeFolderPath, null); + } + + /** + * Only runs the specified test. Pass null or omit to run all tests. + */ + protected Iterable testFolder(String relativeFolderPath, String testToRun) { var testFiles = ResourceHelper.findRelativeResourceFiles(relativeFolderPath, getClass()); @@ -36,6 +45,11 @@ protected Iterable testFolder(String relativeFolderPath) var testFilePath = Path.of(testFile); var testFileName = testFilePath.getFileName().toString(); + if (testToRun != null && !testFileName.equals(testToRun)) + { + return Stream.of(); + } + var fileType = testFileName.contains(".") ? NaturalFileType.fromExtension(testFileName.split("\\.")[1]) : NaturalFileType.SUBPROGRAM; diff --git a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java index 095c93883..de1f82981 100644 --- a/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java +++ b/libs/natparse/src/test/java/org/amshove/natparse/parsing/StatementListParserShould.java @@ -1011,7 +1011,14 @@ void parseWriteWithReportSpecification() void parseWriteWithAttributeDefinition() { var write = assertParsesSingleStatement("WRITE (AD=UL AL=17 NL=8)", IWriteNode.class); - assertThat(write.descendants()).hasSize(10); + assertThat(write.descendants()).hasSize(6); + } + + @Test + void notParseAttributeAsIsnParameter() + { + var write = assertParsesSingleStatement("WRITE *ISN(NL=8)", IWriteNode.class); + assertThat(write.descendants()).anyMatch(n -> n instanceof ITokenNode tNode && tNode.token().kind() == SyntaxKind.NL); } @Test @@ -1894,4 +1901,93 @@ void parseReduceDynamicWithErrorNr() var stmt = assertParsesSingleStatement("REDUCE DYNAMIC #VAR TO 20 GIVING #ERR", IReduceDynamicNode.class); assertIsVariableReference(stmt.errorVariable(), "#ERR"); } + + @Test + void parseDefinePrototype() + { + var prototype = assertParsesSingleStatement(""" + DEFINE PROTOTYPE HI RETURNS (L) + END-PROTOTYPE + """, IDefinePrototypeNode.class); + + assertThat(prototype.nameToken().symbolName()).isEqualTo("HI"); + assertThat(prototype.isVariable()).isFalse(); + assertThat(prototype.variableReference()).isNull(); + } + + @Test + void parseDefinePrototypeVariable() + { + var prototype = assertParsesSingleStatement(""" + DEFINE PROTOTYPE VARIABLE HI RETURNS (L) + END-PROTOTYPE + """, IDefinePrototypeNode.class); + + assertThat(prototype.nameToken().symbolName()).isEqualTo("HI"); + assertThat(prototype.isVariable()).isTrue(); + assertThat(prototype.variableReference()).isNotNull(); + } + + @Test + void parseWritePcWithVariable() + { + var write = assertParsesSingleStatement("WRITE PC FILE 1 VARIABLE 'Hi'", IWritePcNode.class); + assertThat(write.isVariable()).isTrue(); + assertLiteral(write.number(), SyntaxKind.NUMBER_LITERAL); + } + + @Test + void parseWritePcWithoutVariable() + { + var write = assertParsesSingleStatement("WRITE PC FILE 1 'Hi'", IWritePcNode.class); + assertThat(write.isVariable()).isFalse(); + assertLiteral(write.operand(), SyntaxKind.STRING_LITERAL); + } + + @Test + void parseWritePcCommandSync() + { + assertParsesSingleStatement("WRITE PC 5 COMMAND 'Hi' SYNC", IWritePcNode.class); + } + + @Test + void parseWritePcCommandAsync() + { + assertParsesSingleStatement("WRITE PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); + } + + @Test + void parseDownloadPcWithVariable() + { + var download = assertParsesSingleStatement("DOWNLOAD PC FILE 1 VARIABLE 'Hi'", IWritePcNode.class); + assertThat(download.isVariable()).isTrue(); + assertLiteral(download.number(), SyntaxKind.NUMBER_LITERAL); + } + + @Test + void parseDownloadPcWithoutVariable() + { + var download = assertParsesSingleStatement("DOWNLOAD PC FILE 1 'Hi'", IWritePcNode.class); + assertThat(download.isVariable()).isFalse(); + assertLiteral(download.operand(), SyntaxKind.STRING_LITERAL); + } + + @Test + void parseDownloadPcCommandSync() + { + assertParsesSingleStatement("DOWNLOAD PC 5 COMMAND 'Hi' SYNC", IWritePcNode.class); + } + + @Test + void parseDownloadPcCommandAsync() + { + assertParsesSingleStatement("DOWNLOAD PC 5 COMMAND 'Hi' ASYNC", IWritePcNode.class); + } + + @Test + void allowLabelIdentifierAsVariableOperand() + { + var assignment = assertParsesSingleStatement("#VAR(R1.) := 5", IAssignmentStatementNode.class); + assertIsVariableReference(assignment.target(), "#VAR"); + } } diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables new file mode 100644 index 000000000..d2374941a --- /dev/null +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/regressiontests/writeThinksFlagsAreVariables @@ -0,0 +1,12 @@ +DEFINE DATA +LOCAL +END-DEFINE + +/* These should all not raise any diagnostics, because no variables are referenced +WRITE NOTITLE NOHDR USING FORM 'DAFORM' +WRITE NOTITLE NOHDR USING MAP 'DAMAP' +WRITE NOTITLE NOHDR USING MAP 'DAMAP' NO PARAMETER +WRITE (4) 'HI' +WRITE TITLE LEFT JUSTIFIED UNDERLINED 'Hi' SKIP 2 LINES + +END diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests new file mode 100644 index 000000000..2a9230825 --- /dev/null +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/arrayAccessTests @@ -0,0 +1,58 @@ +DEFINE DATA +PARAMETER +1 #P-ARR (A40/1:*) OPTIONAL +LOCAL +1 #NOT-ARRAY (A10) +1 #ARR (A10/*) +1 #I (I4) +1 #GRP +2 #GRPARR1 (A10/*) +2 #GRPARR2 (A10/*) +1 THEVIEW VIEW OF THE-DDM +2 PERIODIC-GRP +3 INPER1 (A10/*) +3 INPER2 (A10/*) +END-DEFINE + +WRITE #NOT-ARRAY(DY='021I'01) /* No diagnostic, DY is an attribute +WRITE #NOT-ARRAY(HIS.) /* No diagnostic, label reference + +/* No diagnostics, array accessed +WRITE #ARR(*) +WRITE #ARR(5) + +/* No dimension specified +WRITE #ARR /* !{D:ERROR:NPP042} + +/* Shouldn't have a dimension +WRITE #NOT-ARRAY(5) /* !{D:ERROR:NPP042} + +/* Wrong dimensions specified +WRITE #ARR(*, *) /* !{D:ERROR:NPP042} +WRITE #ARR(5, 1) /* !{D:ERROR:NPP042} + +/* No diagnostic, *OCC needs actual array +#I := *OCC(#ARR) +#I := *OCCURRENCE(#ARR) + +/* No diagnostic, resizing statements need actual array +RESIZE ARRAY #ARR TO (1:10) +REDUCE ARRAY #ARR TO (1:5) +EXPAND ARRAY #ARR TO (1:8) + +/* No array access allowed +WRITE #GRP(*) /* !{D:ERROR:NPP042} +/* Array access allowed because its a periodic group +WRITE PERIODIC-GRP(*) +/* Array access needed because its a periodic group +WRITE PERIODIC-GRP /* !{D:ERROR:NPP042} + +IF #P-ARR SPECIFIED /* IF SPECIFIED doesn't need to access index + IGNORE +END-IF + +/* Bounds need actual array, like OCC +#I := *UBOUND(#ARR, 1) +#I := *LBOUND(#ARR, 1) + +END diff --git a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds index 7f59882dc..5de63e1a6 100644 --- a/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds +++ b/libs/natparse/src/test/resources/org/amshove/natparse/parsing/typing/writeWorkDynamicArrayBounds @@ -1,6 +1,7 @@ DEFINE DATA LOCAL 1 #ARR (A10/*) +1 #ARR3D (A10/*,*,*) 1 #I (I4) 1 #J (I4) END-DEFINE @@ -14,13 +15,13 @@ WRITE WORK FILE 1 #ARR(1:5) WRITE WORK FILE 1 #ARR(#I) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,#J) +WRITE WORK FILE 1 #ARR3D(2,#I,#J) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,#J) +WRITE WORK FILE 1 #ARR3D(2,#I,#J) /* This should raise no diagnostic, because direct index access with variable is fine -WRITE WORK FILE 1 #ARR(2,#I,*) +WRITE WORK FILE 1 #ARR3D(2,#I,*) /* This should raise no diagnostic, because it is no range access WRITE WORK FILE 1 #ARR(*) diff --git a/tools/ruletranslator/src/main/resources/rules/NPP042 b/tools/ruletranslator/src/main/resources/rules/NPP042 new file mode 100644 index 000000000..3f9af4ccc --- /dev/null +++ b/tools/ruletranslator/src/main/resources/rules/NPP042 @@ -0,0 +1,6 @@ +name: Invalid array access +priority: BLOCKER +tags: compile-time +type: BUG +description: +This reference to a variable either needs a dimension specified (because the target variable is an array) or needs toe have the dimension removed (because the target is not an array).