provide test coverage reporting within the build
This commit is contained in:
parent
d20b49dc90
commit
894f7ca210
GPG key ID:
A7B62D657EF764F8
3 changed files with 185 additions and 143 deletions
|
|
@ -1,6 +1,6 @@
|
|||
const std = @import("std");
|
||||
const GitVersion = @import("build/GitVersion.zig");
|
||||
const coverage = @import("build/coverage.zig");
|
||||
const Coverage = @import("build/Coverage.zig");
|
||||
|
||||
pub fn build(b: *std.Build) void {
|
||||
const target = b.standardTargetOptions(.{});
|
||||
|
|
@ -151,8 +151,8 @@ pub fn build(b: *std.Build) void {
|
|||
test_step.dependOn(&run_tests.step);
|
||||
|
||||
// Coverage step
|
||||
const cov_exe = coverage.addCoverageStep(b, root_module);
|
||||
configureCompilationUnit(cov_exe, libs);
|
||||
const cov_step = Coverage.addCoverageStep(b, root_module, exe.name);
|
||||
configureCompilationUnit(cov_step.test_exe, libs);
|
||||
}
|
||||
|
||||
fn configureCompilationUnit(compile: *std.Build.Step.Compile, libs: []const *std.Build.Step.Compile) void {
|
||||
|
|
|
|||
182
build/Coverage.zig
Normal file
182
build/Coverage.zig
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
const builtin = @import("builtin");
|
||||
const std = @import("std");
|
||||
const Build = std.Build;
|
||||
|
||||
const Coverage = @This();
|
||||
|
||||
/// Adds test coverage. This will create a new test coverage executable to the
|
||||
/// build graph, generated only if coverage is a target. It will create an
|
||||
/// option -Dcoverage-threshold that will fail the build if the threshold is
|
||||
/// not met. It will also add a step that downloads a zig fork of the kcov
|
||||
/// executable into zig cache if it doesn't already exist
|
||||
///
|
||||
/// Because it is creating a new test executable from the root module provided,
|
||||
/// if there are any linking steps being done to your test executable, those
|
||||
/// must also be done to the test_exe returned by this function
|
||||
pub fn addCoverageStep(b: *Build, root_module: *Build.Module, coverage_name: []const u8) *Coverage {
|
||||
//verify host requirements
|
||||
{
|
||||
const supported = builtin.os.tag == .linux and
|
||||
(builtin.cpu.arch == .x86_64 or builtin.cpu.arch == .aarch64);
|
||||
|
||||
if (!supported)
|
||||
@panic("Coverage only supported on x86_64-linux or aarch64-linux");
|
||||
}
|
||||
|
||||
// Add options
|
||||
const coverage_threshold = b.option(u7, "coverage-threshold", "Minimum coverage percentage required") orelse 0;
|
||||
const coverage_dir = b.option([]const u8, "coverage-dir", "Coverage output directory") orelse
|
||||
b.pathJoin(&.{ b.build_root.path orelse ".", "coverage" });
|
||||
const coverage_step = b.step("coverage", "Generate test coverage report");
|
||||
|
||||
// Set up kcov download
|
||||
// We can't do it directly because we are sandboxed during build, but
|
||||
// we can create a program and run that program. First we need the destination
|
||||
// directory
|
||||
const kcov = blk: {
|
||||
const arch_name = switch (builtin.cpu.arch) {
|
||||
.x86_64 => "x86_64",
|
||||
.aarch64 => "aarch64",
|
||||
else => unreachable,
|
||||
};
|
||||
|
||||
const Algo = std.crypto.hash.sha2.Sha256;
|
||||
var hasher = Algo.init(.{});
|
||||
hasher.update("kcov-");
|
||||
hasher.update(arch_name);
|
||||
var cache_hash: [Algo.digest_length]u8 = undefined;
|
||||
hasher.final(&cache_hash);
|
||||
|
||||
const cache_dir = b.pathJoin(&.{
|
||||
b.cache_root.path.?,
|
||||
"o",
|
||||
b.fmt("{s}", .{std.fmt.bytesToHex(cache_hash, .lower)}),
|
||||
});
|
||||
|
||||
const kcov_name = b.fmt("kcov-{s}", .{arch_name});
|
||||
break :blk .{ .path = b.pathJoin(&.{ cache_dir, kcov_name }), .arch = arch_name };
|
||||
};
|
||||
|
||||
// Create download and coverage build steps
|
||||
return blk: {
|
||||
const download_exe = b.addExecutable(.{
|
||||
.name = "download-kcov",
|
||||
.root_module = b.createModule(.{
|
||||
.root_source_file = b.path("build/download_kcov.zig"),
|
||||
.target = b.resolveTargetQuery(.{}),
|
||||
}),
|
||||
});
|
||||
|
||||
const run_download = b.addRunArtifact(download_exe);
|
||||
run_download.addArg(kcov.path);
|
||||
run_download.addArg(kcov.arch);
|
||||
|
||||
const run_coverage = b.addSystemCommand(&.{kcov.path});
|
||||
const include_path = b.pathJoin(&.{ b.build_root.path.?, "src" });
|
||||
run_coverage.addArgs(&.{ "--include-path", include_path });
|
||||
const css_file = b.pathJoin(&.{ b.build_root.path.?, "build", "bcov.css" });
|
||||
run_coverage.addArg(b.fmt("--configure=css-file={s}", .{css_file}));
|
||||
run_coverage.addArg(coverage_dir);
|
||||
const test_exe = b.addTest(.{
|
||||
.name = coverage_name,
|
||||
.root_module = root_module,
|
||||
// we need to set the test exe to use llvm as the self hosted backend
|
||||
// does not support the data kcov needs
|
||||
.use_llvm = true,
|
||||
});
|
||||
run_coverage.addArtifactArg(test_exe);
|
||||
run_coverage.step.dependOn(&test_exe.step);
|
||||
run_coverage.step.dependOn(&run_download.step);
|
||||
|
||||
const json_path = b.fmt("{s}/{s}/coverage.json", .{ coverage_dir, coverage_name });
|
||||
const verbose = b.option(bool, "coverage-verbose", "Show test coverage for each file") orelse false;
|
||||
const check_step = create(b, test_exe, json_path, coverage_threshold, verbose);
|
||||
check_step.step.dependOn(&run_coverage.step);
|
||||
coverage_step.dependOn(&check_step.step);
|
||||
break :blk check_step;
|
||||
};
|
||||
}
|
||||
|
||||
step: Build.Step,
|
||||
json_path: []const u8,
|
||||
threshold: u7,
|
||||
test_exe: *std.Build.Step.Compile,
|
||||
verbose: bool,
|
||||
|
||||
pub fn create(owner: *Build, test_exe: *std.Build.Step.Compile, xml_path: []const u8, threshold: u7, verbose: bool) *Coverage {
|
||||
const check = owner.allocator.create(Coverage) catch @panic("OOM");
|
||||
check.* = .{
|
||||
.step = Build.Step.init(.{
|
||||
.id = .custom,
|
||||
.name = "check coverage",
|
||||
.owner = owner,
|
||||
.makeFn = make,
|
||||
}),
|
||||
.json_path = xml_path,
|
||||
.threshold = threshold,
|
||||
.test_exe = test_exe,
|
||||
.verbose = verbose,
|
||||
};
|
||||
return check;
|
||||
}
|
||||
|
||||
// This must be kept in step with kcov coverage.json format
|
||||
const CoverageReport = struct {
|
||||
percent_covered: f64,
|
||||
covered_lines: usize,
|
||||
total_lines: usize,
|
||||
percent_low: u7,
|
||||
percent_high: u7,
|
||||
command: []const u8,
|
||||
date: []const u8,
|
||||
files: []File,
|
||||
};
|
||||
|
||||
const File = struct {
|
||||
file: []const u8,
|
||||
percent_covered: f64,
|
||||
covered_lines: usize,
|
||||
total_lines: usize,
|
||||
|
||||
pub fn coverageLessThanDesc(context: []File, lhs: File, rhs: File) bool {
|
||||
_ = context;
|
||||
return lhs.percent_covered > rhs.percent_covered;
|
||||
}
|
||||
};
|
||||
|
||||
fn make(step: *Build.Step, options: Build.Step.MakeOptions) !void {
|
||||
_ = options;
|
||||
const check: *Coverage = @fieldParentPtr("step", step);
|
||||
const allocator = step.owner.allocator;
|
||||
|
||||
const file = try std.fs.cwd().openFile(check.json_path, .{});
|
||||
defer file.close();
|
||||
|
||||
const content = try file.readToEndAlloc(allocator, 10 * 1024 * 1024);
|
||||
defer allocator.free(content);
|
||||
|
||||
const json = try std.json.parseFromSlice(CoverageReport, allocator, content, .{});
|
||||
defer json.deinit();
|
||||
const coverage = json.value;
|
||||
|
||||
var stdout_buffer: [1024]u8 = undefined;
|
||||
var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer);
|
||||
const stdout = &stdout_writer.interface;
|
||||
if (step.owner.verbose or check.verbose) {
|
||||
const files = coverage.files;
|
||||
std.mem.sort(File, files, files, File.coverageLessThanDesc);
|
||||
for (files) |f|
|
||||
try stdout.print(
|
||||
"{d: >5.1}% {d: >5}/{d: <5}:{s}\n",
|
||||
.{ f.percent_covered, f.covered_lines, f.total_lines, f.file },
|
||||
);
|
||||
}
|
||||
try stdout.print(
|
||||
"Total test coverage: {d}% ({d}/{d})\n",
|
||||
.{ coverage.percent_covered, coverage.covered_lines, coverage.total_lines },
|
||||
);
|
||||
try stdout.flush();
|
||||
|
||||
if (@as(u7, @intFromFloat(@floor(coverage.percent_covered))) < check.threshold)
|
||||
return step.fail("Coverage {d}% is below threshold {d}%", .{ coverage.percent_covered, check.threshold });
|
||||
}
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
const builtin = @import("builtin");
|
||||
const std = @import("std");
|
||||
const Build = std.Build;
|
||||
|
||||
pub fn addCoverageStep(b: *Build, root_module: *Build.Module) *Build.Step.Compile {
|
||||
//verify host requirements
|
||||
{
|
||||
const supported = builtin.os.tag == .linux and
|
||||
(builtin.cpu.arch == .x86_64 or builtin.cpu.arch == .aarch64);
|
||||
|
||||
if (!supported)
|
||||
@panic("Coverage only supported on x86_64-linux or aarch64-linux");
|
||||
}
|
||||
|
||||
// Add options
|
||||
const coverage_threshold = b.option(u8, "coverage-threshold", "Minimum coverage percentage required") orelse 0;
|
||||
const coverage_dir = b.option([]const u8, "coverage-dir", "Coverage output directory") orelse
|
||||
b.pathJoin(&.{ b.build_root.path orelse ".", "coverage" });
|
||||
const coverage_step = b.step("coverage", "Generate test coverage report");
|
||||
|
||||
// Set up kcov download
|
||||
// We can't do it directly because we are sandboxed during build, but
|
||||
// we can create a program and run that program. First we need the destination
|
||||
// directory
|
||||
const kcov = blk: {
|
||||
const arch_name = switch (builtin.cpu.arch) {
|
||||
.x86_64 => "x86_64",
|
||||
.aarch64 => "aarch64",
|
||||
else => unreachable,
|
||||
};
|
||||
|
||||
const Algo = std.crypto.hash.sha2.Sha256;
|
||||
var hasher = Algo.init(.{});
|
||||
hasher.update("kcov-");
|
||||
hasher.update(arch_name);
|
||||
var cache_hash: [Algo.digest_length]u8 = undefined;
|
||||
hasher.final(&cache_hash);
|
||||
|
||||
const cache_dir = b.pathJoin(&.{
|
||||
b.cache_root.path.?,
|
||||
"o",
|
||||
b.fmt("{s}", .{std.fmt.bytesToHex(cache_hash, .lower)}),
|
||||
});
|
||||
|
||||
const kcov_name = b.fmt("kcov-{s}", .{arch_name});
|
||||
break :blk .{ .path = b.pathJoin(&.{ cache_dir, kcov_name }), .arch = arch_name };
|
||||
};
|
||||
|
||||
// Create download and coverage build steps
|
||||
return blk: {
|
||||
const download_exe = b.addExecutable(.{
|
||||
.name = "download-kcov",
|
||||
.root_module = b.createModule(.{
|
||||
.root_source_file = b.path("build/download_kcov.zig"),
|
||||
.target = b.resolveTargetQuery(.{}),
|
||||
}),
|
||||
});
|
||||
|
||||
const run_download = b.addRunArtifact(download_exe);
|
||||
run_download.addArg(kcov.path);
|
||||
run_download.addArg(kcov.arch);
|
||||
|
||||
const run_coverage = b.addSystemCommand(&.{kcov.path});
|
||||
run_coverage.addArgs(&.{ "--include-path", "src/" });
|
||||
const css_file = b.pathJoin(&.{ b.build_root.path.?, "build", "bcov.css" });
|
||||
run_coverage.addArg(b.fmt("--configure=css-file={s}", .{css_file}));
|
||||
run_coverage.addArg(coverage_dir);
|
||||
const test_exe = b.addTest(.{
|
||||
.root_module = root_module,
|
||||
// we need to set the test exe to use llvm as the self hosted backend
|
||||
// does not support the data kcov needs
|
||||
.use_llvm = true,
|
||||
});
|
||||
run_coverage.addArtifactArg(test_exe);
|
||||
run_coverage.step.dependOn(&test_exe.step);
|
||||
run_coverage.step.dependOn(&run_download.step);
|
||||
|
||||
if (coverage_threshold > 0) {
|
||||
const xml_path = b.fmt("{s}/test/cobertura.xml", .{coverage_dir});
|
||||
const check_step = CheckCoverage.create(b, xml_path, coverage_threshold);
|
||||
check_step.step.dependOn(&run_coverage.step);
|
||||
coverage_step.dependOn(&check_step.step);
|
||||
} else {
|
||||
coverage_step.dependOn(&run_coverage.step);
|
||||
}
|
||||
break :blk test_exe;
|
||||
};
|
||||
}
|
||||
|
||||
pub const CheckCoverage = struct {
|
||||
step: Build.Step,
|
||||
xml_path: []const u8,
|
||||
threshold: u8,
|
||||
|
||||
pub fn create(owner: *Build, xml_path: []const u8, threshold: u8) *CheckCoverage {
|
||||
const check = owner.allocator.create(CheckCoverage) catch @panic("OOM");
|
||||
check.* = .{
|
||||
.step = Build.Step.init(.{
|
||||
.id = .custom,
|
||||
.name = "check coverage",
|
||||
.owner = owner,
|
||||
.makeFn = make,
|
||||
}),
|
||||
.xml_path = xml_path,
|
||||
.threshold = threshold,
|
||||
};
|
||||
return check;
|
||||
}
|
||||
|
||||
fn make(step: *Build.Step, options: Build.Step.MakeOptions) !void {
|
||||
_ = options;
|
||||
const check: *CheckCoverage = @fieldParentPtr("step", step);
|
||||
const allocator = step.owner.allocator;
|
||||
|
||||
const file = try std.fs.cwd().openFile(check.xml_path, .{});
|
||||
defer file.close();
|
||||
|
||||
const content = try file.readToEndAlloc(allocator, 10 * 1024 * 1024);
|
||||
defer allocator.free(content);
|
||||
|
||||
const needle = "line-rate=\"";
|
||||
const start = std.mem.indexOf(u8, content, needle) orelse return error.CoverageNotFound;
|
||||
const value_start = start + needle.len;
|
||||
const value_end = std.mem.indexOfScalarPos(u8, content, value_start, '"') orelse return error.CoverageNotFound;
|
||||
const value_str = content[value_start..value_end];
|
||||
|
||||
const coverage = try std.fmt.parseFloat(f64, value_str);
|
||||
const coverage_int: u8 = @intFromFloat(coverage * 100.0);
|
||||
|
||||
if (step.owner.verbose) {
|
||||
var stdout_buffer: [1024]u8 = undefined;
|
||||
var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer);
|
||||
const stdout = &stdout_writer.interface;
|
||||
try stdout.print("Coverage: {d}%\n", .{coverage_int});
|
||||
}
|
||||
|
||||
if (coverage_int < check.threshold)
|
||||
return step.fail("Coverage {d}% is below threshold {d}%", .{ coverage_int, check.threshold });
|
||||
}
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue