Add JaCoCo coverage dump command
JaCoCo coverage can be used to create reports helping to identify fuzz
blockers. The --coverage_dump flag can now specify a dump file
containing coverage information that should be created after the fuzz
run.
diff --git a/README.md b/README.md
index 90d2baa..fdceec9 100644
--- a/README.md
+++ b/README.md
@@ -471,6 +471,33 @@
Particular stack traces can also be ignored based on their `DEDUP_TOKEN` by passing a comma-separated list of tokens
via `--ignore=<token_1>,<token2>`.
+### Export coverage information
+
+The internally gathered JaCoCo coverage information can be exported in a human-readable and the JaCoCo dump format.
+These can help identify code areas that can not be reached through fuzzing and perhaps need changes to make them more
+accessible for the fuzzer.
+
+The human-readable report contains coverage information, like branch and line coverage, on file level. It's useful to
+get a quick overview about the overall coverage. The flag `--coverage_report=<file>` can be used to generate the report.
+
+Similar to the JaCoCo `dump` command the flag `--coverage_dump=<file>` specifies the coverage dump file, often called
+`coverage.exec`, that should be generated after the fuzzing run. It contains a binary representation of the gathered
+coverage data in the JaCoCo format.
+
+The JaCoCo `report` command can be used to generate reports based on the coverage dump. For example the
+following command generates an HTML report in the folder `./report/` containing all classes available in `classes.jar`
+and their coverage as captured in the export `coverage.exec`.
+```shell
+java -jar jacococli.jar report coverage.exec \
+ --classfiles classes.jar \
+ --sourcefiles some/path/to/sources \
+ --html ./report/ \
+ --name FuzzCoverageReport
+```
+
+More information about coverage report generation is available on the JaCoCo
+[CLI documentation](https://www.eclemma.org/jacoco/trunk/doc/cli.html) page.
+
## Advanced fuzz targets
### Fuzzing with Native Libraries
diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt
index b873d11..7ad5908 100644
--- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt
+++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt
@@ -19,14 +19,13 @@
import io.github.classgraph.ClassGraph
import org.jacoco.core.analysis.CoverageBuilder
import org.jacoco.core.data.ExecutionData
-import org.jacoco.core.data.ExecutionDataReader
import org.jacoco.core.data.ExecutionDataStore
import org.jacoco.core.data.ExecutionDataWriter
import org.jacoco.core.data.SessionInfo
-import org.jacoco.core.data.SessionInfoStore
import org.jacoco.core.internal.data.CRC64
-import java.io.ByteArrayInputStream
-import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.FileOutputStream
+import java.io.OutputStream
import java.time.Instant
import java.util.UUID
@@ -65,8 +64,18 @@
CoverageMap.replayCoveredIds(additionalCoverage)
}
+ /**
+ * [dumpCoverageReport] dumps a human-readable coverage report of files using any [coveredIds] to [dumpFileName].
+ */
@JvmStatic
- fun computeFileCoverage(coveredIds: IntArray): String {
+ fun dumpCoverageReport(coveredIds: IntArray, dumpFileName: String) {
+ File(dumpFileName).bufferedWriter().use { writer ->
+ writer.write(computeFileCoverage(coveredIds))
+ }
+ }
+
+ private fun computeFileCoverage(coveredIds: IntArray): String {
+ fun Double.format(digits: Int) = "%.${digits}f".format(this)
val coverage = analyzeCoverage(coveredIds.toSet()) ?: return "No classes were instrumented"
return coverage.sourceFiles.joinToString(
"\n",
@@ -104,21 +113,42 @@
}
}
- private fun Double.format(digits: Int) = "%.${digits}f".format(this)
+ /**
+ * [dumpJacocoCoverage] dumps the JaCoCo coverage of files using any [coveredIds] to [dumpFileName].
+ * JaCoCo only exports coverage for files containing at least one coverage data point. The dump
+ * can be used by the JaCoCo report command to create reports also including not covered files.
+ */
+ @JvmStatic
+ fun dumpJacocoCoverage(coveredIds: IntArray, dumpFileName: String) {
+ FileOutputStream(dumpFileName).use { outStream ->
+ dumpJacocoCoverage(coveredIds, outStream)
+ }
+ }
- fun dumpJacocoCoverage(coveredIds: Set<Int>): ByteArray? {
+ /**
+ * [dumpJacocoCoverage] dumps the JaCoCo coverage of files using any [coveredIds] to [outStream].
+ */
+ @JvmStatic
+ fun dumpJacocoCoverage(coveredIds: IntArray, outStream: OutputStream) {
+ // Return if no class has been instrumented.
+ val startTimestamp = startTimestamp ?: return
+
// Update the list of covered IDs with the coverage information for the current run.
updateCoveredIdsWithCoverageMap()
val dumpTimestamp = Instant.now()
- val outStream = ByteArrayOutputStream()
val outWriter = ExecutionDataWriter(outStream)
- // Return null if no class has been instrumented.
- val startTimestamp = startTimestamp ?: return null
outWriter.visitSessionInfo(
SessionInfo(UUID.randomUUID().toString(), startTimestamp.epochSecond, dumpTimestamp.epochSecond)
)
+ analyzeJacocoCoverage(coveredIds.toSet()).accept(outWriter)
+ }
+ /**
+ * Build up a JaCoCo [ExecutionDataStore] based on [coveredIds] containing the internally gathered coverage information.
+ */
+ private fun analyzeJacocoCoverage(coveredIds: Set<Int>): ExecutionDataStore {
+ val executionDataStore = ExecutionDataStore()
val sortedCoveredIds = (additionalCoverage + coveredIds).sorted().toIntArray()
for ((internalClassName, info) in instrumentedClassInfo) {
// Determine the subarray of coverage IDs in sortedCoveredIds that contains the IDs generated while
@@ -148,25 +178,19 @@
.forEach { classLocalEdgeId ->
probes[classLocalEdgeId] = true
}
- outWriter.visitClassExecution(ExecutionData(info.classId, internalClassName, probes))
+ executionDataStore.visitClassExecution(ExecutionData(info.classId, internalClassName, probes))
}
- return outStream.toByteArray()
+ return executionDataStore
}
+ /**
+ * Create a [CoverageBuilder] containing all classes matching the include/exclude pattern and their coverage statistics.
+ */
fun analyzeCoverage(coveredIds: Set<Int>): CoverageBuilder? {
return try {
val coverage = CoverageBuilder()
analyzeAllUncoveredClasses(coverage)
- val rawExecutionData = dumpJacocoCoverage(coveredIds) ?: return null
- val executionDataStore = ExecutionDataStore()
- val sessionInfoStore = SessionInfoStore()
- ByteArrayInputStream(rawExecutionData).use { stream ->
- ExecutionDataReader(stream).run {
- setExecutionDataVisitor(executionDataStore)
- setSessionInfoVisitor(sessionInfoStore)
- read()
- }
- }
+ val executionDataStore = analyzeJacocoCoverage(coveredIds)
for ((internalClassName, info) in instrumentedClassInfo) {
EdgeCoverageInstrumentor(ClassInstrumentor.defaultEdgeCoverageStrategy, ClassInstrumentor.defaultCoverageMap, 0)
.analyze(
@@ -194,7 +218,6 @@
.asSequence()
.map { it.replace('/', '.') }
.toSet()
- val emptyExecutionDataStore = ExecutionDataStore()
ClassGraph()
.enableClassInfo()
.ignoreClassVisibility()
@@ -205,6 +228,9 @@
"jaz",
)
.scan().use { result ->
+ // ExecutionDataStore is used to look up existing coverage during analysis of the class files,
+ // no entries are added during that. Passing in an empty store is fine for uncovered files.
+ val emptyExecutionDataStore = ExecutionDataStore()
result.allClasses
.asSequence()
.filter { classInfo -> classNameGlobber.includes(classInfo.name) }
diff --git a/driver/coverage_tracker.cpp b/driver/coverage_tracker.cpp
index edfde5c..4be0db1 100644
--- a/driver/coverage_tracker.cpp
+++ b/driver/coverage_tracker.cpp
@@ -114,7 +114,7 @@
AssertNoException(env);
}
-std::string CoverageTracker::ComputeCoverage(JNIEnv &env) {
+void CoverageTracker::ReportCoverage(JNIEnv &env, std::string report_file) {
uintptr_t *covered_pcs;
size_t num_covered_pcs = __sanitizer_cov_get_observed_pcs(&covered_pcs);
std::vector<jint> covered_edge_ids(covered_pcs,
@@ -123,24 +123,43 @@
jclass coverage_recorder = env.FindClass(kCoverageRecorderClass);
AssertNoException(env);
- jmethodID coverage_recorder_compute_file_coverage = env.GetStaticMethodID(
- coverage_recorder, "computeFileCoverage", "([I)Ljava/lang/String;");
+ jmethodID coverage_recorder_dump_coverage_report = env.GetStaticMethodID(
+ coverage_recorder, "dumpCoverageReport", "([ILjava/lang/String;)V");
AssertNoException(env);
jintArray covered_edge_ids_jni = env.NewIntArray(num_covered_pcs);
AssertNoException(env);
env.SetIntArrayRegion(covered_edge_ids_jni, 0, num_covered_pcs,
covered_edge_ids.data());
AssertNoException(env);
- auto file_coverage_jni = (jstring)(env.CallStaticObjectMethod(
- coverage_recorder, coverage_recorder_compute_file_coverage,
- covered_edge_ids_jni));
+ jstring report_file_str = env.NewStringUTF(report_file.c_str());
+ env.CallStaticObjectMethod(coverage_recorder,
+ coverage_recorder_dump_coverage_report,
+ covered_edge_ids_jni, report_file_str);
AssertNoException(env);
- auto file_coverage_cstr = env.GetStringUTFChars(file_coverage_jni, nullptr);
+}
+
+void CoverageTracker::DumpCoverage(JNIEnv &env, std::string dump_file) {
+ uintptr_t *covered_pcs;
+ size_t num_covered_pcs = __sanitizer_cov_get_observed_pcs(&covered_pcs);
+ std::vector<jint> covered_edge_ids(covered_pcs,
+ covered_pcs + num_covered_pcs);
+ delete[] covered_pcs;
+
+ jclass coverage_recorder = env.FindClass(kCoverageRecorderClass);
AssertNoException(env);
- std::string file_coverage(file_coverage_cstr);
- env.ReleaseStringUTFChars(file_coverage_jni, file_coverage_cstr);
+ jmethodID coverage_recorder_dump_jacoco_coverage = env.GetStaticMethodID(
+ coverage_recorder, "dumpJacocoCoverage", "([ILjava/lang/String;)V");
AssertNoException(env);
- return file_coverage;
+ jintArray covered_edge_ids_jni = env.NewIntArray(num_covered_pcs);
+ AssertNoException(env);
+ env.SetIntArrayRegion(covered_edge_ids_jni, 0, num_covered_pcs,
+ covered_edge_ids.data());
+ AssertNoException(env);
+ jstring dump_file_str = env.NewStringUTF(dump_file.c_str());
+ env.CallStaticObjectMethod(coverage_recorder,
+ coverage_recorder_dump_jacoco_coverage,
+ covered_edge_ids_jni, dump_file_str);
+ AssertNoException(env);
}
} // namespace jazzer
diff --git a/driver/coverage_tracker.h b/driver/coverage_tracker.h
index d5b80a0..455fe15 100644
--- a/driver/coverage_tracker.h
+++ b/driver/coverage_tracker.h
@@ -44,6 +44,7 @@
static void RecordInitialCoverage(JNIEnv &env);
static void ReplayInitialCoverage(JNIEnv &env);
- static std::string ComputeCoverage(JNIEnv &env);
+ static void ReportCoverage(JNIEnv &env, std::string);
+ static void DumpCoverage(JNIEnv &env, std::string);
};
} // namespace jazzer
diff --git a/driver/fuzz_target_runner.cpp b/driver/fuzz_target_runner.cpp
index 10d9052..d7e2d7c 100644
--- a/driver/fuzz_target_runner.cpp
+++ b/driver/fuzz_target_runner.cpp
@@ -62,6 +62,9 @@
DEFINE_string(coverage_report, "",
"Path at which a coverage report is stored when the fuzzer "
"exits. If left empty, no report is generated (default)");
+DEFINE_string(coverage_dump, "",
+ "Path at which a coverage dump is stored when the fuzzer "
+ "exits. If left empty, no dump is generated (default)");
DEFINE_string(autofuzz, "",
"Fully qualified reference to a method on the classpath that "
@@ -262,20 +265,18 @@
FuzzTargetRunner::~FuzzTargetRunner() {
if (FLAGS_hooks && !FLAGS_coverage_report.empty()) {
- std::string report = CoverageTracker::ComputeCoverage(jvm_.GetEnv());
- std::ofstream report_file(FLAGS_coverage_report);
- if (report_file) {
- report_file << report << std::flush;
- } else {
- LOG(ERROR) << "Failed to write coverage report to "
- << FLAGS_coverage_report;
- }
+ CoverageTracker::ReportCoverage(jvm_.GetEnv(), FLAGS_coverage_report);
+ }
+ if (FLAGS_hooks && !FLAGS_coverage_dump.empty()) {
+ CoverageTracker::DumpCoverage(jvm_.GetEnv(), FLAGS_coverage_dump);
}
if (fuzzer_tear_down_ != nullptr) {
std::cerr << "calling fuzzer teardown function" << std::endl;
jvm_.GetEnv().CallStaticVoidMethod(jclass_, fuzzer_tear_down_);
- if (jthrowable exception = jvm_.GetEnv().ExceptionOccurred())
+ if (jthrowable exception = jvm_.GetEnv().ExceptionOccurred()) {
std::cerr << getStackTrace(exception) << std::endl;
+ _Exit(1);
+ }
}
}
diff --git a/driver/libfuzzer_driver.cpp b/driver/libfuzzer_driver.cpp
index d4f8803..c3deec3 100644
--- a/driver/libfuzzer_driver.cpp
+++ b/driver/libfuzzer_driver.cpp
@@ -45,6 +45,9 @@
// Defined in fuzz_target_runner.cpp
DECLARE_string(coverage_report);
+// Defined in fuzz_target_runner.cpp
+DECLARE_string(coverage_dump);
+
// This symbol is defined by sanitizers if linked into Jazzer or in
// sanitizer_symbols.cpp if no sanitizer is used.
extern "C" void __sanitizer_set_death_callback(void (*)());
@@ -129,6 +132,11 @@
"fuzzing and has been disabled";
FLAGS_coverage_report = "";
}
+ if (!FLAGS_coverage_dump.empty()) {
+ LOG(WARNING) << "WARN: --coverage_dump does not support parallel "
+ "fuzzing and has been disabled";
+ FLAGS_coverage_dump = "";
+ }
if (FLAGS_id_sync_file.empty()) {
// Create an empty temporary file used for coverage ID synchronization and
// pass its path to the agent in every child process. This requires adding
diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel
index 2870ede..19ccac2 100644
--- a/tests/BUILD.bazel
+++ b/tests/BUILD.bazel
@@ -77,12 +77,19 @@
srcs = [
"src/test/java/com/example/CoverageFuzzer.java",
],
+ env = {
+ "COVERAGE_REPORT_FILE": "coverage.txt",
+ "COVERAGE_DUMP_FILE": "coverage.exec",
+ },
fuzzer_args = [
"-use_value_profile=1",
- "--coverage_report=coverage.exec",
+ "--coverage_report=coverage.txt",
+ "--coverage_dump=coverage.exec",
+ "--instrumentation_includes=com.example.**",
],
target_class = "com.example.CoverageFuzzer",
verify_crash_input = False,
+ verify_crash_reproducer = False,
deps = [
"@jazzer_jacoco//:jacoco_internal",
],
diff --git a/tests/src/test/java/com/example/CoverageFuzzer.java b/tests/src/test/java/com/example/CoverageFuzzer.java
index e3a4880..3312970 100644
--- a/tests/src/test/java/com/example/CoverageFuzzer.java
+++ b/tests/src/test/java/com/example/CoverageFuzzer.java
@@ -18,16 +18,48 @@
import com.code_intelligence.jazzer.api.FuzzedDataProvider;
import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow;
+import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.jacoco.core.data.ExecutionData;
+import org.jacoco.core.data.ExecutionDataReader;
+import org.jacoco.core.data.ExecutionDataStore;
+import org.jacoco.core.data.SessionInfoStore;
+/**
+ * Test of coverage report and dump.
+ *
+ * Internally, JaCoCo is used to gather coverage information to guide the fuzzer to cover new
+ * branches. This information can be dumped in the JaCoCo format and used to generate reports later
+ * on. The dump only contains classes with at least one coverage data point. A JaCoCo report will
+ * also include completely uncovered files based on the available classes in the stated jar files
+ * in the report command.
+ *
+ * A human-readable coverage report can be generated directly by Jazzer. It contains information
+ * on file level about all classes that should have been instrumented according to the
+ * instrumentation_includes and instrumentation_exclude filters.
+ */
@SuppressWarnings({"unused", "UnusedReturnValue"})
public final class CoverageFuzzer {
+ // Not used during fuzz run, so not included in the dump
+ public static class ClassNotToCover {
+ private final int i;
+ public ClassNotToCover(int i) {
+ this.i = i;
+ }
+ public int getI() {
+ return i;
+ }
+ }
+
+ // Used in the fuzz run and included in the dump
public static class ClassToCover {
private final int i;
@@ -58,67 +90,99 @@
}
public static void fuzzerTearDown() throws IOException {
- List<String> coverage = Files.readAllLines(Paths.get("./coverage.exec"));
- assertEquals(871, coverage.size());
+ assertCoverageReport();
+ assertCoverageDump();
+ }
+ private static void assertCoverageReport() throws IOException {
+ List<String> coverage = Files.readAllLines(Paths.get(System.getenv("COVERAGE_REPORT_FILE")));
List<List<String>> sections = new ArrayList<>(4);
sections.add(new ArrayList<>());
coverage.forEach(l -> {
if (l.isEmpty()) {
sections.add(new ArrayList<>());
+ } else {
+ sections.get(sections.size() - 1).add(l);
}
- sections.get(sections.size() - 1).add(l);
});
List<String> branchCoverage = sections.get(0);
- assertEquals(217, branchCoverage.size());
+ assertEquals(2, branchCoverage.size());
List<String> lineCoverage = sections.get(1);
- assertEquals(218, lineCoverage.size());
+ assertEquals(2, lineCoverage.size());
List<String> incompleteCoverage = sections.get(2);
- assertEquals(218, incompleteCoverage.size());
+ assertEquals(2, incompleteCoverage.size());
List<String> missedCoverage = sections.get(3);
- assertEquals(218, missedCoverage.size());
+ assertEquals(2, missedCoverage.size());
- String branch =
+ assertNotNull(
branchCoverage.stream()
.filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
.findFirst()
- .orElseThrow(() -> new IllegalStateException("Could not find branch coverage"));
- // assertEquals("CoverageFuzzer.java: 11/16 (68.75%)", branch);
+ .orElseThrow(() -> new IllegalStateException("Could not find branch coverage")));
- String line = lineCoverage.stream()
- .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
- .findFirst()
- .orElseThrow(() -> new IllegalStateException("Could not find line coverage"));
- assertEquals("CoverageFuzzer.java: 15/61 (24.59%)", line);
+ assertNotNull(
+ lineCoverage.stream()
+ .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("Could not find line coverage")));
- String incomplete =
+ assertNotNull(
incompleteCoverage.stream()
.filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
.findFirst()
- .orElseThrow(() -> new IllegalStateException("Could not find incomplete coverage"));
- assertEquals("CoverageFuzzer.java: []", incomplete);
+ .orElseThrow(() -> new IllegalStateException("Could not find incomplete coverage")));
String missed =
missedCoverage.stream()
.filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("Could not find missed coverage"));
- if (IntStream.rangeClosed(15, 44).anyMatch(i -> missed.contains(String.valueOf(i)))) {
- throw new IllegalStateException("No coverage collected for ClassToCover");
+ List<String> missingLines = IntStream.rangeClosed(63, 79)
+ .mapToObj(i -> " " + i)
+ .filter(missed::contains)
+ .collect(Collectors.toList());
+ if (!missingLines.isEmpty()) {
+ throw new IllegalStateException(String.format(
+ "Missing coverage for ClassToCover on lines %s", String.join(", ", missingLines)));
+ }
+ }
+
+ private static void assertCoverageDump() throws IOException {
+ ExecutionDataStore executionDataStore = new ExecutionDataStore();
+ SessionInfoStore sessionInfoStore = new SessionInfoStore();
+ try (FileInputStream bais = new FileInputStream(System.getenv("COVERAGE_DUMP_FILE"))) {
+ ExecutionDataReader reader = new ExecutionDataReader(bais);
+ reader.setExecutionDataVisitor(executionDataStore);
+ reader.setSessionInfoVisitor(sessionInfoStore);
+ reader.read();
+ }
+ assertEquals(2, executionDataStore.getContents().size());
+
+ ExecutionData coverageFuzzerCoverage = new ExecutionData(0, "", 0);
+ ExecutionData classToCoverCoverage = new ExecutionData(0, "", 0);
+ for (ExecutionData content : executionDataStore.getContents()) {
+ if (content.getName().endsWith("ClassToCover")) {
+ classToCoverCoverage = content;
+ } else {
+ coverageFuzzerCoverage = content;
+ }
}
- // TODO switch to JaCoCo coverage report format
- // CoverageBuilder coverage = new CoverageBuilder();
- // ExecutionDataStore executionDataStore = new ExecutionDataStore();
- // SessionInfoStore sessionInfoStore = new SessionInfoStore();
- // try (FileInputStream bais = new FileInputStream("./coverage.exec")) {
- // ExecutionDataReader reader = new ExecutionDataReader(bais);
- // reader.setExecutionDataVisitor(executionDataStore);
- // reader.setSessionInfoVisitor(sessionInfoStore);
- // reader.read();
- // }
- // System.out.println(coverage.getClasses());
+ assertEquals("com/example/CoverageFuzzer", coverageFuzzerCoverage.getName());
+ assertEquals(6, countHits(coverageFuzzerCoverage.getProbes()));
+
+ assertEquals("com/example/CoverageFuzzer$ClassToCover", classToCoverCoverage.getName());
+ assertEquals(11, countHits(classToCoverCoverage.getProbes()));
+ }
+
+ private static int countHits(boolean[] probes) {
+ int count = 0;
+ for (boolean probe : probes) {
+ if (probe)
+ count++;
+ }
+ return count;
}
private static <T> void assertEquals(T expected, T actual) {
@@ -127,4 +191,10 @@
String.format("Expected \"%s\", got \"%s\"", expected, actual));
}
}
+
+ private static <T> void assertNotNull(T actual) {
+ if (actual == null) {
+ throw new IllegalStateException("Expected none null value, got null");
+ }
+ }
}