Skip to content

feat: exclude warmup rounds from perf flamegraph #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
not-matthias merged 3 commits into from
Apr 21, 2026
Merged

feat: exclude warmup rounds from perf flamegraph #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f133367
build(jmh-fork): exclude native C sources from license plugin
not-matthias Apr 17, 2026
47b11fe
feat(jmh-fork): emit benchmark markers around measurement iterations,…
not-matthias Apr 17, 2026
ff83f3b
fix(jmh-fork): make InstrumentHooks.currentTimestamp safe when native…
not-matthias Apr 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion jmh-fork/jmh-core/native-instrument-hooks/src/main/c/instrument_hooks_jni.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ static void throw_runtime_exception(JNIEnv *env, const char *msg) {
// --- Static methods ---

JNIEXPORT jlong JNICALL
Java_io_codspeed_instrument_1hooks_InstrumentHooks_currentTimestamp(
Java_io_codspeed_instrument_1hooks_InstrumentHooks_nativeCurrentTimestamp(
JNIEnv *env, jclass cls) {
(void)env;
(void)cls;
Expand Down
1 change: 1 addition & 0 deletions jmh-fork/jmh-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ questions.
<excludes>
<exclude>src/main/java/io/codspeed/**</exclude>
<exclude>src/test/java/io/codspeed/**</exclude>
<exclude>native-*/**</exclude>
</excludes>
</configuration>
</execution>
Expand Down
6 changes: 5 additions & 1 deletion jmh-fork/jmh-core/src/main/java/io/codspeed/instrument_hooks/InstrumentHooks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@ public static InstrumentHooks getInstance() {
public static final int FEATURE_DISABLE_CALLGRIND_MARKERS = 0;

// Static methods (no instance needed)
public static native long currentTimestamp();
public static long currentTimestamp() {
return nativeAvailable ? nativeCurrentTimestamp() : 0L;
}

private static native long nativeCurrentTimestamp();

public static native void setFeature(int feature, boolean enabled);

Expand Down
21 changes: 21 additions & 0 deletions jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@
*/
abstract class BaseRunner {

/**
* System property the parent Runner sets on forked JVMs whose iterations are warmup-only
* (results discarded). The forked process reads it to suppress CodSpeed marker emission so the
* runner only sees real measurement regions.
*/
static final String CODSPEED_WARMUP_FORK_PROP = "io.codspeed.warmupFork";

static boolean isWarmupFork() {
return "true".equals(System.getProperty(CODSPEED_WARMUP_FORK_PROP));
}

private long projectedTotalTime;
private long projectedRunningTime;
private long actualRunningTime;
Expand Down Expand Up @@ -291,7 +302,17 @@ protected void runBenchmark(BenchmarkParams benchParams, BenchmarkHandler handle

boolean isFirstIteration = (benchParams.getWarmup().getCount() == 0) && (i == 1);
boolean isLastIteration = (i == mp.getCount());

long startTs = InstrumentHooks.currentTimestamp();
IterationResult ir = handler.runIteration(benchParams, mp, isFirstIteration, isLastIteration);
long endTs = InstrumentHooks.currentTimestamp();

if (!isWarmupFork()) {
InstrumentHooks hooks = InstrumentHooks.getInstance();
Comment thread
not-matthias marked this conversation as resolved.
int pid = (int) ProcessHandle.current().pid();
hooks.addMarker(pid, InstrumentHooks.MARKER_TYPE_BENCHMARK_START, startTs);
hooks.addMarker(pid, InstrumentHooks.MARKER_TYPE_BENCHMARK_END, endTs);
}
out.iterationResult(benchParams, mp, i, ir);

allMeasurement += ir.getMetadata().getAllOps();
Expand Down
21 changes: 16 additions & 5 deletions jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -681,7 +681,7 @@ private Multimap<BenchmarkParams, BenchmarkResult> runSeparate(ActionPlan action

for (int i = 0; i < totalForks; i++) {
boolean warmupFork = (i < warmupForkCount);
List<String> forkedString = getForkedMainCommand(params, profilers, server.getHost(), server.getPort());
List<String> forkedString = getForkedMainCommand(params, profilers, server.getHost(), server.getPort(), warmupFork);

etaBeforeBenchmark();

Expand Down Expand Up @@ -844,11 +844,18 @@ private List<IterationResult> doFork(BinaryLinkServer reader, List<String> comma
}

/**
* @param host host VM host
* @param port host VM port
* @return
* Build the command line used to launch a forked benchmark JVM.
*
* @param benchmark benchmark parameters for the current run
* @param profilers external profilers contributing extra JVM args
* @param host host VM host
* @param port host VM port
* @param warmupFork true for warmup-only forks whose results are discarded; causes
* {@code -D}{@value BaseRunner#CODSPEED_WARMUP_FORK_PROP}{@code =true}
* to be added so the forked JVM suppresses CodSpeed marker emission
* @return the full argv, starting with the JVM executable
*/
List<String> getForkedMainCommand(BenchmarkParams benchmark, List<ExternalProfiler> profilers, String host, int port) {
List<String> getForkedMainCommand(BenchmarkParams benchmark, List<ExternalProfiler> profilers, String host, int port, boolean warmupFork) {
// Poll profilers for options
List<String> javaInvokeOptions = new ArrayList<>();
List<String> javaOptions = new ArrayList<>();
Expand Down Expand Up @@ -876,6 +883,10 @@ List<String> getForkedMainCommand(BenchmarkParams benchmark, List<ExternalProfil
}
}

if (warmupFork) {
command.add("-D" + CODSPEED_WARMUP_FORK_PROP + "=true");
}

// add profiler JVM commands, if any profiler wants it
command.addAll(javaOptions);

Expand Down
6 changes: 3 additions & 3 deletions jmh-fork/jmh-core/src/test/java/org/openjdk/jmh/runner/RunnerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public void testEmptyOptsHaveCompileCommandFile() {
Utils.getCurrentJvm(), Collections.<String>emptyList(),
System.getProperty("java.version"), System.getProperty("java.vm.name"), System.getProperty("java.vm.version"), Version.getPlainVersion(),
TimeValue.days(1));
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT);
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT, false);

// expecting 1 compile command file
List<String> files = CompilerHints.getCompileCommandFiles(command);
Expand Down Expand Up @@ -100,7 +100,7 @@ public void testOptsWithCompileCommandFileResultInMergedCompileCommandFile() thr
Utils.getCurrentJvm(), Collections.singletonList(CompilerHints.XX_COMPILE_COMMAND_FILE + tempHints),
System.getProperty("java.version"), System.getProperty("java.vm.name"), System.getProperty("java.vm.version"), Version.getPlainVersion(),
TimeValue.days(1));
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT);
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT, false);

// expecting 1 compile command file
List<String> files = CompilerHints.getCompileCommandFiles(command);
Expand Down Expand Up @@ -137,7 +137,7 @@ public void testOptsWith2CompileCommandFilesResultInMergedCompileCommandFile() t
Arrays.asList(CompilerHints.XX_COMPILE_COMMAND_FILE + tempHints1, CompilerHints.XX_COMPILE_COMMAND_FILE + tempHints2),
System.getProperty("java.version"), System.getProperty("java.vm.name"), System.getProperty("java.vm.version"), Version.getPlainVersion(),
TimeValue.days(1));
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT);
List<String> command = blade.getForkedMainCommand(bp, Collections.<ExternalProfiler>emptyList(), DUMMY_HOST, DUMMY_PORT, false);

// expecting 1 compile command file
List<String> files = CompilerHints.getCompileCommandFiles(command);
Expand Down
Loading