From 2f73ec3bc8ebb821e1e4e0fa8ffa458ec1fc454d Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 13 Jun 2024 11:43:50 +0200 Subject: [PATCH 1/4] Reformat coverage and move export to command --- src/main/java/com/askimed/nf/test/App.java | 9 +- .../nf/test/commands/CoverageCommand.java | 89 +++++++++++++++++++ .../nf/test/commands/RunTestsCommand.java | 18 ++-- .../nf/test/lang/dependencies/Coverage.java | 80 ++++++++++++++++- .../lang/dependencies/CoverageItemSorter.java | 9 ++ 5 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/askimed/nf/test/commands/CoverageCommand.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index ba7a7e26..d9db5d1e 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -1,12 +1,6 @@ package com.askimed.nf.test; -import com.askimed.nf.test.commands.CleanCommand; -import com.askimed.nf.test.commands.GenerateTestsCommand; -import com.askimed.nf.test.commands.InitCommand; -import com.askimed.nf.test.commands.ListTestsCommand; -import com.askimed.nf.test.commands.RunTestsCommand; -import com.askimed.nf.test.commands.UpdatePluginsCommand; -import com.askimed.nf.test.commands.VersionCommand; +import com.askimed.nf.test.commands.*; import ch.qos.logback.classic.Level; import picocli.CommandLine; @@ -35,6 +29,7 @@ public int run(String[] args) { commandLine.addSubcommand("clean", new CleanCommand()); commandLine.addSubcommand("init", new InitCommand()); commandLine.addSubcommand("test", new RunTestsCommand()); + commandLine.addSubcommand("coverage", new CoverageCommand()); commandLine.addSubcommand("list", new ListTestsCommand()); commandLine.addSubcommand("ls", new ListTestsCommand()); commandLine.addSubcommand("generate", new GenerateTestsCommand()); diff --git a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java new file mode 100644 index 00000000..8e9e6365 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java @@ -0,0 +1,89 @@ +package com.askimed.nf.test.commands; + +import com.askimed.nf.test.config.Config; +import com.askimed.nf.test.lang.dependencies.Coverage; +import com.askimed.nf.test.lang.dependencies.DependencyResolver; +import com.askimed.nf.test.util.AnsiColors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Help.Visibility; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@Command(name = "coverage") +public class CoverageCommand extends AbstractCommand { + + private static final String SHARD_STRATEGY_ROUND_ROBIN = "round-robin"; + + @Option(names = { + "--csv" }, description = "Write coverage results in csv format", required = false, showDefaultValue = Visibility.ALWAYS) + private String csv = null; + + @Option(names = { "--config", + "-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS) + private String configFilename = Config.FILENAME; + + private static Logger log = LoggerFactory.getLogger(CoverageCommand.class); + + @Override + public Integer execute() throws Exception { + + List scripts = new ArrayList(); + Config config = null; + + try { + + File defaultConfigFile = null; + boolean defaultWithTrace = true; + try { + File configFile = new File(configFilename); + if (configFile.exists()) { + log.info("Load config from file {}...", configFile.getAbsolutePath()); + config = Config.parse(configFile); + } else { + System.out.println(AnsiColors.yellow("Warning: This pipeline has no nf-test config file.")); + log.warn("No nf-test config file found."); + } + + } catch (Exception e) { + + System.out.println(AnsiColors.red("Error: Syntax errors in nf-test config file: " + e)); + log.error("Parsing config file failed", e); + return 2; + + } + + File baseDir = new File(new File("").getAbsolutePath()); + DependencyResolver resolver = new DependencyResolver(baseDir); + resolver.setFollowingDependencies(true); + + + if (config != null) { + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + } else { + resolver.buildGraph(); + } + + Coverage coverage = new Coverage(resolver).getAll(); + if (csv != null) { + coverage.exportAsCsv(csv); + } else { + coverage.printDetails(); + } + + return 0; + + } catch (Throwable e) { + + System.out.println(AnsiColors.red("Error: " + e));log.error("Running tests failed.", e); + return 1; + + } + + } + +} diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index ab8e8e2a..e43a7e08 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -188,7 +188,6 @@ public Integer execute() throws Exception { return 2; } - List ignorePatterns = new Vector(); File baseDir = new File(new File("").getAbsolutePath()); DependencyResolver resolver = new DependencyResolver(baseDir); resolver.setFollowingDependencies(followDependencies); @@ -231,10 +230,6 @@ public Integer execute() throws Exception { AnsiText.printBulletList(scripts); - if (coverage) { - new Coverage(resolver).getByFiles(testPaths).print(); - } - } else { if (config != null) { resolver.buildGraph(config.getIgnore(), config.getTriggers()); @@ -242,9 +237,6 @@ public Integer execute() throws Exception { resolver.buildGraph(); } scripts = resolver.findTestsByFiles(testPaths); - if (coverage) { - new Coverage(resolver).getAll().print(); - } } if (graph != null) { @@ -304,7 +296,15 @@ public Integer execute() throws Exception { System.out.println(AnsiColors.yellow("Dry run mode activated: tests are not executed, just listed.")); } - return engine.execute(); + int exitStatus = engine.execute(); + + if (coverage && findRelatedTests) { + new Coverage(resolver).getByFiles(testPaths).print(); + } else if (coverage) { + new Coverage(resolver).getAll().print(); + } + + return exitStatus; } catch (Throwable e) { diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java index 3d38cf62..60700821 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java @@ -1,10 +1,18 @@ package com.askimed.nf.test.lang.dependencies; +import com.askimed.nf.test.core.TestExecutionResult; +import com.askimed.nf.test.core.TestSuiteExecutionResult; +import com.askimed.nf.test.core.reports.CsvReportWriter; import com.askimed.nf.test.util.AnsiColors; +import com.askimed.nf.test.util.AnsiText; +import com.opencsv.CSVWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; import java.text.DecimalFormat; import java.util.List; import java.util.Vector; @@ -19,12 +27,15 @@ public class Coverage { private static Logger log = LoggerFactory.getLogger(Coverage.class); + private File baseDir = null; + public Coverage(DependencyGraph graph) { this.graph = graph; } public Coverage(DependencyResolver resolver) { this.graph = resolver.getGraph(); + baseDir = resolver.getBaseDir(); } public void add(File file, boolean covered) { @@ -57,6 +68,8 @@ public Coverage getAll(){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -87,6 +100,8 @@ public Coverage getByFiles(List files){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -95,14 +110,71 @@ public Coverage getByFiles(List files){ } public void print() { - DecimalFormat decimalFormat = new DecimalFormat("#.##"); + printLabel(); + System.out.println(); + } + + public void printDetails() { System.out.println(); - System.out.print("Coverage: " + getCoveredItems() + "/" + getItems().size()); - System.out.println(" (" + decimalFormat.format(getCoveredItems() / (float) getItems().size() * 100) + "%)"); + System.out.println("Files:"); for (Coverage.CoverageItem item : getItems()) { - System.out.println(" - " + (item.isCovered() ? AnsiColors.green(item.getFile().getAbsolutePath()) : AnsiColors.red(item.getFile().getAbsolutePath()))); + String label = item.getFile().getAbsolutePath(); + if (baseDir != null) { + label = Paths.get(baseDir.getAbsolutePath()).relativize(item.getFile().toPath()).toString(); + } + System.out.println(" \u2022 " + (item.isCovered() ? AnsiColors.green(label) : AnsiColors.red(label))); } System.out.println(); + printLabel(); + System.out.println(); + } + + private void printLabel() { + float coverage = getCoveredItems() / (float) getItems().size(); + System.out.print(getColor("COVERAGE:", coverage) + " " + formatCoverage(coverage)); + System.out.println( " [" + getCoveredItems() + " of " + getItems().size() + " files]"); + } + + private String getColor(String label, float value) { + if (value < 0.5) { + return AnsiColors.red(label); + } else if (value < 0.9) { + return AnsiColors.yellow(label); + } else { + return AnsiColors.green(label); + } + } + + private String formatCoverage(float value) { + DecimalFormat decimalFormat = new DecimalFormat("#.##"); + return decimalFormat.format(value * 100) + "%"; + } + + public void exportAsCsv(String filename) throws IOException { + String[] header = new String[]{ + "filename", + "covered", + "type" + }; + + CSVWriter writer = new CSVWriter(new FileWriter(new File(filename))); + writer.writeNext(header); + for (Coverage.CoverageItem item : getItems()) { + String[] line = new String[]{ + item.getFile().getAbsolutePath(), + item.isCovered() + "", + "unknown" + }; + + writer.writeNext(line); + } + + writer.close(); + System.out.println(); + printLabel(); + System.out.println(); + System.out.println("Wrote coverage report to file " + filename + "\n"); + } public static class CoverageItem { diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java new file mode 100644 index 00000000..4c7b1fd3 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java @@ -0,0 +1,9 @@ +package com.askimed.nf.test.lang.dependencies; + +public class CoverageItemSorter implements java.util.Comparator { + + @Override + public int compare(Coverage.CoverageItem o1, Coverage.CoverageItem o2) { + return o1.getFile().getAbsolutePath().compareTo(o2.getFile().getAbsolutePath()); + } +} From 5bfd0889e0d037a7c37f5ef406c24959ff3f4d3c Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 13 Jun 2024 14:18:29 +0200 Subject: [PATCH 2/4] Add simple html coverage report --- .../nf/test/commands/CoverageCommand.java | 7 +++ .../nf/test/lang/dependencies/Coverage.java | 41 +++++++++++++--- .../lang/dependencies/coverage-report.html | 48 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html diff --git a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java index 8e9e6365..757962c4 100644 --- a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java @@ -23,6 +23,11 @@ public class CoverageCommand extends AbstractCommand { "--csv" }, description = "Write coverage results in csv format", required = false, showDefaultValue = Visibility.ALWAYS) private String csv = null; + @Option(names = { + "--html" }, description = "Write coverage results in html format", required = false, showDefaultValue = Visibility.ALWAYS) + private String html = null; + + @Option(names = { "--config", "-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS) private String configFilename = Config.FILENAME; @@ -71,6 +76,8 @@ public Integer execute() throws Exception { Coverage coverage = new Coverage(resolver).getAll(); if (csv != null) { coverage.exportAsCsv(csv); + } else if (html != null) { + coverage.exportAsHtml(html); } else { coverage.printDetails(); } diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java index 60700821..cd85c6ea 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java @@ -1,24 +1,31 @@ package com.askimed.nf.test.lang.dependencies; +import com.askimed.nf.test.commands.init.InitTemplates; import com.askimed.nf.test.core.TestExecutionResult; import com.askimed.nf.test.core.TestSuiteExecutionResult; import com.askimed.nf.test.core.reports.CsvReportWriter; import com.askimed.nf.test.util.AnsiColors; import com.askimed.nf.test.util.AnsiText; +import com.askimed.nf.test.util.FileUtil; import com.opencsv.CSVWriter; +import groovy.lang.Writable; +import groovy.text.SimpleTemplateEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.net.URL; import java.nio.file.Paths; import java.text.DecimalFormat; -import java.util.List; -import java.util.Vector; +import java.text.DecimalFormatSymbols; +import java.util.*; public class Coverage { + private static final String HTML_TEMPLATE = "coverage-report.html"; + private int coveredItems = 0; private DependencyGraph graph; @@ -118,10 +125,7 @@ public void printDetails() { System.out.println(); System.out.println("Files:"); for (Coverage.CoverageItem item : getItems()) { - String label = item.getFile().getAbsolutePath(); - if (baseDir != null) { - label = Paths.get(baseDir.getAbsolutePath()).relativize(item.getFile().toPath()).toString(); - } + String label = getFileLabel(item.getFile()); System.out.println(" \u2022 " + (item.isCovered() ? AnsiColors.green(label) : AnsiColors.red(label))); } System.out.println(); @@ -129,12 +133,24 @@ public void printDetails() { System.out.println(); } + public String getFileLabel(File file) { + String label = file.getAbsolutePath(); + if (baseDir != null) { + label = Paths.get(baseDir.getAbsolutePath()).relativize(file.toPath()).toString(); + } + return label; + } + private void printLabel() { float coverage = getCoveredItems() / (float) getItems().size(); System.out.print(getColor("COVERAGE:", coverage) + " " + formatCoverage(coverage)); System.out.println( " [" + getCoveredItems() + " of " + getItems().size() + " files]"); } + public float getCoverage() { + return getCoveredItems() / (float) getItems().size(); + } + private String getColor(String label, float value) { if (value < 0.5) { return AnsiColors.red(label); @@ -146,7 +162,7 @@ private String getColor(String label, float value) { } private String formatCoverage(float value) { - DecimalFormat decimalFormat = new DecimalFormat("#.##"); + DecimalFormat decimalFormat = new DecimalFormat("#.##", DecimalFormatSymbols.getInstance(Locale.US)); return decimalFormat.format(value * 100) + "%"; } @@ -177,12 +193,23 @@ public void exportAsCsv(String filename) throws IOException { } + public void exportAsHtml(String filename) throws IOException, ClassNotFoundException { + Map binding = new HashMap(); + binding.put("coverage", this); + URL templateUrl = Coverage.class.getResource(HTML_TEMPLATE); + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + Writable template = engine.createTemplate(templateUrl).make(binding); + FileUtil.write(new File(filename), template); + } + public static class CoverageItem { private File file; private boolean covered = false; + //TODO: add number of tests?? + public CoverageItem(File file, boolean covered) { this.file = file; this.covered = covered; diff --git a/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html new file mode 100644 index 00000000..331b0120 --- /dev/null +++ b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html @@ -0,0 +1,48 @@ + + + + + + + + + + +
+

Coverage Report

+

This report was generated by nf-test on <%= new Date() %>.

+

+ Coverage: <%= coverage.formatCoverage(coverage.getCoverage()) %> +

+ +
+
+
+ +
+ + + + + + + + + + <% coverage.items.each { item -> %> + + + + + <% } %> + +
FileCovered
<%= coverage.getFileLabel(item.getFile()) %><%= item.isCovered() %>
+
+ + \ No newline at end of file From f7b4d41b30d85798aa0f9ebf39a3360ad41f783e Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 13 Jun 2024 14:58:18 +0200 Subject: [PATCH 3/4] Fix coverage badge --- .../com/askimed/nf/test/lang/dependencies/coverage-report.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html index 331b0120..d0ad39b6 100644 --- a/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html +++ b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html @@ -18,7 +18,7 @@

Coverage Report

This report was generated by nf-test on <%= new Date() %>.

- Coverage: <%= coverage.formatCoverage(coverage.getCoverage()) %> + Coverage: <%= coverage.formatCoverage(coverage.getCoverage()) %>

From 6aba845bc79b60854252db7d327d17e97454c5da Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 13 Jun 2024 16:01:16 +0200 Subject: [PATCH 4/4] Add basic documentation --- docs/docs/cli/coverage.md | 19 +++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 20 insertions(+) create mode 100644 docs/docs/cli/coverage.md diff --git a/docs/docs/cli/coverage.md b/docs/docs/cli/coverage.md new file mode 100644 index 00000000..2d29f111 --- /dev/null +++ b/docs/docs/cli/coverage.md @@ -0,0 +1,19 @@ +# `coverage` command + +:octicons-tag-24: 0.9.0 + +## Usage + +``` +nf-test coverage +``` + +The `coverage` command prints information about the number of Nextflow files that are covered by a test. + +### Optional Arguments + +#### `--csv ` +Writes a coverage report in csv format. + +#### `--html ` +Writes a coverage report in html format. diff --git a/mkdocs.yml b/mkdocs.yml index 29829fd3..9667650f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ nav: - generate: docs/cli/generate.md - test: docs/cli/test.md - list: docs/cli/list.md + - coverage: docs/cli/coverage.md - clean: docs/cli/clean.md - Configuration: docs/configuration.md - Plugins: