From dc3919f6f272e0d28471f5c5ae9eeebe26710f81 Mon Sep 17 00:00:00 2001 From: Rohan Padhye Date: Mon, 22 Mar 2021 12:18:48 -0400 Subject: [PATCH] Internal: Get handle to fuzz guidance in JQF test runner Closes #82 --- .../java/edu/berkeley/cs/jqf/fuzz/JQF.java | 60 +++++++++++++++++-- .../fuzz/junit/quickcheck/FuzzStatement.java | 42 +------------ 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java index 672a81ac3..1cae507ed 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java @@ -28,6 +28,8 @@ */ package edu.berkeley.cs.jqf.fuzz; +import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Random; @@ -36,7 +38,12 @@ import com.pholser.junit.quickcheck.internal.generator.ServiceLoaderGeneratorSource; import com.pholser.junit.quickcheck.random.SourceOfRandomness; import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; +import edu.berkeley.cs.jqf.fuzz.guidance.Guidance; +import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException; +import edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing; import edu.berkeley.cs.jqf.fuzz.junit.quickcheck.FuzzStatement; +import edu.berkeley.cs.jqf.fuzz.random.NoGuidance; +import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.InitializationError; import org.junit.runners.model.Statement; @@ -56,8 +63,7 @@ public JQF(Class clazz) throws InitializationError { super(clazz); // Initialize generator repository with a deterministic seed (for reproducibility) SourceOfRandomness randomness = new SourceOfRandomness(new Random(42)); - this.generatorRepository = new GeneratorRepository(randomness).register(new ServiceLoaderGeneratorSource());; - + this.generatorRepository = new GeneratorRepository(randomness).register(new ServiceLoaderGeneratorSource()); } @Override protected List computeTestMethods() { @@ -83,9 +89,53 @@ private void validateFuzzMethods(List errors) { } @Override public Statement methodBlock(FrameworkMethod method) { - if (method.getAnnotation(Fuzz.class) != null) { - return new FuzzStatement(method, getTestClass(), generatorRepository); + // JQF only needs special handling for @Fuzz-annotated test methods + if (method.getAnnotation(Fuzz.class) == null) { + return super.methodBlock(method); } - return super.methodBlock(method); + + // Get currently set fuzzing guidance + Guidance guidance = GuidedFuzzing.getCurrentGuidance(); + + // If nothing is set, default to random or repro + if (guidance == null) { + // Check for @Fuzz(repro=) + String repro = method.getAnnotation(Fuzz.class).repro(); + if (repro.isEmpty()) { + guidance = new NoGuidance(GuidedFuzzing.DEFAULT_MAX_TRIALS, System.err); + } else { + String reproPath; + // Check if repro path is variable (e.g. `${foo}`) + if (repro.matches("\\$\\{[a-zA-Z.\\d_$]*\\}")) { + // Get a system property with that name (e.g. `foo`) + String key = repro.substring(2, repro.length()-1); + String val = System.getProperty(key); + + // Check if such a property is set + if (val == null) { + throw new IllegalArgumentException(String.format("Test method has " + + "@Fuzz annotation with repro=%s, but such a system " + + "property is not set. Use `-D%s=` when running.", + repro, key)); + } + + reproPath = val; + } else { + // If it is not a variable, then treat it literally + reproPath = repro; + } + + // Create a ReproGuidance with the given path + File inputFile = new File(reproPath); + try { + guidance = new ReproGuidance(inputFile, null); + } catch (IOException e) { + throw new GuidanceException(String.format("Could not open repro file: %s", + inputFile.getAbsolutePath()), e); + } + } + } + + return new FuzzStatement(method, getTestClass(), generatorRepository, guidance); } } diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 27db87dfa..e4e5e49e8 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -78,9 +78,10 @@ public class FuzzStatement extends Statement { private final GeneratorRepository generatorRepository; private final List> expectedExceptions; private final List failures = new ArrayList<>(); + private final Guidance guidance; public FuzzStatement(FrameworkMethod method, TestClass testClass, - GeneratorRepository generatorRepository) { + GeneratorRepository generatorRepository, Guidance fuzzGuidance) { this.method = method; this.testClass = testClass; this.typeVariables = @@ -89,7 +90,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass, .genericsMap(); this.generatorRepository = generatorRepository; this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes()); - + this.guidance = fuzzGuidance; } @@ -106,43 +107,6 @@ public void evaluate() throws Throwable { .map(this::produceGenerator) .collect(Collectors.toList()); - // Get the currently registered fuzz guidance - Guidance guidance = GuidedFuzzing.getCurrentGuidance(); - - // If nothing is set, default to random or repro - if (guidance == null) { - // Check for @Fuzz(repro=) - String repro = method.getAnnotation(Fuzz.class).repro(); - if (repro.isEmpty()) { - guidance = new NoGuidance(GuidedFuzzing.DEFAULT_MAX_TRIALS, System.err); - } else { - String reproPath; - // Check if repro path is variable (e.g. `${foo}`) - if (repro.matches("\\$\\{[a-zA-Z.\\d_$]*\\}")) { - // Get a system property with that name (e.g. `foo`) - String key = repro.substring(2, repro.length()-1); - String val = System.getProperty(key); - - // Check if such a property is set - if (val == null) { - throw new IllegalArgumentException(String.format("Test method has " + - "@Fuzz annotation with repro=%s, but such a system " + - "property is not set. Use `-D%s=` when running.", - repro, key)); - } - - reproPath = val; - } else { - // If it is not a variable, then treat it literally - reproPath = repro; - } - - // Create a ReproGuidance with the given path - File inputFile = new File(reproPath); - guidance = new ReproGuidance(inputFile, null); - } - } - // Keep fuzzing until no more input or I/O error with guidance try {