Skip to content

Commit

Permalink
Merge pull request #205 from MarkusAmshove/natparse/check-array-access
Browse files Browse the repository at this point in the history 
Add checks for array access
  • Loading branch information
@MarkusAmshove
MarkusAmshove authored May 1, 2023
2 parents 20c6b04 + 52f2231 commit d68adba
Show file tree
Hide file tree
Showing 36 changed files with 1,150 additions and 61 deletions.
10 changes: 5 additions & 5 deletions docs/implemented-statements.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ This document tracks the implementation status of Natural statements.

Legend:

:x: - not implemented (65)
:x: - not implemented (64)

:white_check_mark: - implemented or reporting (51)

partial - partially implemented to prevent false positives (6)
partial - partially implemented to prevent false positives (7)

| Statement | Status |
| --- |--------------------|
Expand Down Expand Up @@ -41,21 +41,21 @@ partial - partially implemented to prevent false positives (6)
| DEFINE DATA | :white_check_mark: |
| DEFINE FUNCTION | partial |
| DEFINE PRINTER | :white_check_mark: |
| DEFINE PROTOTYPE | :x: |
| DEFINE PROTOTYPE | partial |
| DEFINE SUBROUTINE | :white_check_mark: |
| DEFINE WINDOW | partial |
| DEFINE WORK FILE | :white_check_mark: |
| DELETE | :x: |
| DELETE (SQL) | :x: |
| DISPLAY | :x: |
| DIVIDE | :white_check_mark: |
| DOWNLOAD PC FILE | :x: |
| DOWNLOAD PC FILE | :white_check_mark: |
| EJECT | :white_check_mark: |
| END | :white_check_mark: |
| END TRANSACTION | :x: |
| ESCAPE | :white_check_mark: |
| EXAMINE | :white_check_mark: |
| EXPAND | :white_check_mark: |
| EXPAND | :white_check_mark: |
| FETCH | :white_check_mark: |
| FIND | :white_check_mark: |
| FOR | :white_check_mark: |
Expand Down
37 changes: 31 additions & 6 deletions libs/natlint/src/main/java/org/amshove/natlint/cli/CliAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;

public class CliAnalyzer
Expand Down Expand Up @@ -93,6 +94,7 @@ public int run()
private final AtomicInteger filesChecked = new AtomicInteger();
private final AtomicInteger totalDiagnostics = new AtomicInteger();
private final AtomicInteger exceptions = new AtomicInteger();
private final AtomicLong linesOfCode = new AtomicLong();

private int analyze(Path projectFilePath)
{
Expand Down Expand Up @@ -172,15 +174,21 @@ private int analyze(Path projectFilePath)
var missTime = missingEndTime - missingStartTime;
var totalTime = indexTime + checkTime + missTime;

var totalTimeSeconds = totalTime / 1000;
System.out.println();
System.out.println("Done.");
System.out.println("Index time: " + indexTime + " ms");
System.out.println("Check time: " + checkTime + " ms");
System.out.println("Miss time : " + missTime + " ms");
System.out.println("Total: " + totalTime + " ms (" + (totalTime / 1000) + "s)");
System.out.println("Files checked: " + filesChecked.get());
System.out.println("Total diagnostics: " + totalDiagnostics.get());
System.out.printf("Index time: %d ms%n", indexTime);
System.out.printf("Check time: %d ms%n", checkTime);
System.out.printf("Miss time : %d ms%n", missTime);
System.out.printf("Total: %d ms (%ds)%n", totalTime, totalTimeSeconds);
System.out.println();
System.out.printf("Files checked: %,d%n", filesChecked.get());
System.out.printf("Lines of code: %,d%n", linesOfCode.get());
System.out.printf("LoC/s: %,d%n", totalTimeSeconds > 0 ? (linesOfCode.get() / totalTimeSeconds) : linesOfCode.get());
System.out.println();
System.out.printf("Total diagnostics: %,d%n", totalDiagnostics.get());
System.out.println("Exceptions: " + exceptions.get());
System.out.println();
System.out.println("Slowest lexed module: " + slowestLexedModule);
System.out.println("Slowest parsed module: " + slowestParsedModule);
System.out.println("Slowest linted module: " + (disableLinting ? "disabled" : slowestLintedModule));
Expand All @@ -203,6 +211,7 @@ private TokenList lex(NaturalFile file, ArrayList<IDiagnostic> allDiagnosticsInF
var lexStart = System.currentTimeMillis();
var tokens = lexer.lex(filesystem.readFile(file.getPath()), file.getPath());
var lexEnd = System.currentTimeMillis();
countLinesOfCode(tokens);
if (slowestLexedModule.milliseconds < lexEnd - lexStart)
{
slowestLexedModule = new SlowestModule(lexEnd - lexStart, file.getProjectRelativePath().toString());
Expand All @@ -221,6 +230,22 @@ private TokenList lex(NaturalFile file, ArrayList<IDiagnostic> allDiagnosticsInF
}
}

private void countLinesOfCode(TokenList tokens)
{
var previousLine = -1;
var totalLines = 0;
for (var token : tokens)
{
if (token.line() != previousLine)
{
totalLines++;
previousLine = token.line();
}
}

linesOfCode.addAndGet(totalLines);
}

private INaturalModule parse(NaturalFile file, TokenList tokens, ArrayList<IDiagnostic> allDiagnosticsInFile)
{
try
Expand Down
17 changes: 17 additions & 0 deletions libs/natls/src/test/java/org/amshove/natls/definition/DefinitionEndpointTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,23 @@ void definitionsShouldReturnTheDefinitionOfVariablesInAssignments()
);
}

@Test
void definitionsShouldReturnTheDefinitionOfVariablesInCVAttribute()
{
assertSingleDefinitionInSameModule(
"""
DEFINE DATA
LOCAL
1 #CVAR (C)
END-DEFINE
WRITE #HI(CV=#C${}$VAR)
END
""",
2, 2
);
}

@Test
void definitionsShouldReturnTheDefinitionOfQualifiedVariablesInAssignments()
{
Expand Down
155 changes: 132 additions & 23 deletions libs/natparse/src/main/java/org/amshove/natparse/lexing/Lexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ private enum LexerMode
{
DEFAULT,
IN_DEFINE_DATA,
IN_DATA_TYPE
}

private LexerMode lexerMode = LexerMode.DEFAULT;
Expand All @@ -55,7 +56,7 @@ public TokenList lex(String source, Path filePath)
continue;
}

if (consumeComment())
if (lexerMode != LexerMode.IN_DATA_TYPE && consumeComment())
{
continue;
}
Expand All @@ -73,10 +74,18 @@ public TokenList lex(String source, Path filePath)
case '(':
inParens = true;
lastBeforeOpenParens = previous();
if (lexerMode == LexerMode.IN_DEFINE_DATA && previous().kind() != SyntaxKind.LESSER_SIGN)
{
lexerMode = LexerMode.IN_DATA_TYPE;
}
createAndAddCurrentSingleToken(SyntaxKind.LPAREN);
continue;
case ')':
inParens = false;
if (lexerMode == LexerMode.IN_DATA_TYPE)
{
lexerMode = LexerMode.IN_DEFINE_DATA;
}
lastBeforeOpenParens = null;
createAndAddCurrentSingleToken(SyntaxKind.RPAREN);
continue;
Expand Down Expand Up @@ -767,7 +776,7 @@ private void consumeIdentifier()
if (scanner.peek() == '/' && scanner.peek(1) == '*')
{
// Slash is a valid character for identifiers, but an asterisk is not.
// If a variable is named #MYVAR/* we can safely assume its a variable followed
// If a variable is named #MYVAR/* we can safely assume it's a variable followed
// by a comment.
break;
}
Expand Down Expand Up @@ -829,28 +838,30 @@ private boolean isValidIdentifierCharacter(char character)

private void consumeIdentifierOrKeyword()
{
if (inParens && scanner.peekText("EM="))
{
editorMask();
return;
}

if (inParens && scanner.peekText("AD="))
{
attributeDefinition();
return;
}

if (inParens && scanner.peekText("CD="))
if (inParens && scanner.peek(2) == '=')
{
colorDefinition();
return;
}
var attributeLookahead = scanner.peekText(3);
var previous = previous();
switch (attributeLookahead)
{
case "AD=" -> attributeDefinition();
case "AL=" -> alphanumericLengthAttribute();
case "CD=" -> colorDefinition();
case "CV=" -> controlVariableAttribute();
case "DF=" -> dateFormatAttribute();
case "DY=" -> dynamicAttribute();
case "EM=" -> editorMask();
case "IP=" -> inputPromptAttribute();
case "IS=" -> identicalSuppressAttribute();
case "NL=" -> numericLengthAttribute();
case "SG=" -> signPosition();
case "ZP=" -> zeroPrintingAttribute();
}

if (inParens && scanner.peekText("DY="))
{
dynamicAttribte();
return;
if (previous() != previous) // check that we consumed something
{
return;
}
}

if (inParens && tokens.size() > 2)
Expand Down Expand Up @@ -900,6 +911,12 @@ private void consumeIdentifierOrKeyword()
break;
}

if (scanner.peek() == '/' && lexerMode == LexerMode.IN_DATA_TYPE)
{
// Slash is a valid character for identifiers, if we're lexing a datatype we can be pretty confident about the slash being for array dimensions
break;
}

if (scanner.peek() == '/')
{ // TODO: this does not work for "KEYWORD/*", eg. END-SUBROUTINE/* bla bla
kindHint = SyntaxKind.IDENTIFIER;
Expand Down Expand Up @@ -987,6 +1004,14 @@ && isValidIdentifierCharacter(scanner.peek()))
}
}

private void controlVariableAttribute()
{
scanner.start();
scanner.advance(3); // CV=
// we intentionally don't consume more, because the variable should be IDENTIFIER
createAndAdd(SyntaxKind.CV);
}

private void editorMask()
{
scanner.start();
Expand Down Expand Up @@ -1030,7 +1055,91 @@ private void attributeDefinition()
createAndAdd(SyntaxKind.AD);
}

private void dynamicAttribte()
private void alphanumericLengthAttribute()
{
scanner.start();
scanner.advance(3); // AL=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.AL);
}

private void zeroPrintingAttribute()
{
scanner.start();
scanner.advance(3); // ZP=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.ZP);
}

private void dateFormatAttribute()
{
scanner.start();
scanner.advance(3); // DF=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.DF);
}

private void inputPromptAttribute()
{
scanner.start();
scanner.advance(3); // IP=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.IP);
}

private void identicalSuppressAttribute()
{
scanner.start();
scanner.advance(3); // IS=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.IS);
}

private void numericLengthAttribute()
{
scanner.start();
scanner.advance(3); // NL=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.NL);
}

private void signPosition()
{
scanner.start();
scanner.advance(3); // SG=
while (!scanner.isAtEnd() && isNoWhitespace() && scanner.peek() != ')')
{
scanner.advance();
}

createAndAdd(SyntaxKind.SG);
}

private void dynamicAttribute()
{
scanner.start();
scanner.advance(3); // DY=
Expand Down
10 changes: 8 additions & 2 deletions libs/natparse/src/main/java/org/amshove/natparse/lexing/SyntaxKind.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

public enum SyntaxKind
{
EOF(false, false, false), // end of file
LBRACKET(false, false, false),
RBRACKET(false, false, false),
LPAREN(false, false, false),
Expand Down Expand Up @@ -107,8 +108,8 @@ public enum SyntaxKind
PROGRAM(true, true, false),
ETID(false, true, false),
CPU_TIME(false, true, false),
LBOUND(false, true, false),
UBOUND(false, true, false),
LBOUND(false, false, true),
UBOUND(false, false, true),
SERVER_TYPE(false, true, false),

// Kcheck reserved keywords
Expand Down Expand Up @@ -758,4 +759,9 @@ public boolean canBeIdentifier()
{
return canBeIdentifier;
}

public boolean isAttribute()
{
return this == AD || this == DY || this == CD || this == EM || this == NL || this == AL || this == DF || this == IP || this == IS || this == CV || this == ZP || this == SG;
}
}
Loading

0 comments on commit d68adba

Please sign in to comment.