diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 0000000000..1831b0a740
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[alias] # command aliases
+ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]
\ No newline at end of file
diff --git a/.github/hyperlight-bot.yml b/.github/hyperlight-bot.yml
new file mode 100644
index 0000000000..758fc3e550
--- /dev/null
+++ b/.github/hyperlight-bot.yml
@@ -0,0 +1,8 @@
+# Configuration for the hyperlight-gh-bot GitHub App.
+# See: https://github.com/jprendes/hyperlight-gh-bot
+
+# Name of the artifact containing the comment body.
+artifact_name: "pr-comment"
+
+# Regex matched against the job name to filter which jobs trigger the bot.
+job_filter: "post-benchmark-comment"
diff --git a/.github/workflows/ValidatePullRequest.yml b/.github/workflows/ValidatePullRequest.yml
index 2f62944768..82d6fc8af2 100644
--- a/.github/workflows/ValidatePullRequest.yml
+++ b/.github/workflows/ValidatePullRequest.yml
@@ -140,6 +140,89 @@ jobs:
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
secrets: inherit
+ # Run benchmarks and post results as PR comment
+ benchmarks:
+ needs:
+ - docs-pr
+ - build-guests
+ # Required because update-guest-locks is skipped on non-dependabot PRs,
+ # and a skipped dependency transitively skips all downstream jobs.
+ # See: https://github.com/actions/runner/issues/2205
+ if: ${{ !cancelled() && !failure() }}
+ strategy:
+ fail-fast: false
+ matrix:
+ hypervisor: ['hyperv-ws2025', mshv3, kvm]
+ cpu: [amd, intel]
+ uses: ./.github/workflows/dep_benchmarks.yml
+ secrets: inherit
+ with:
+ docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+ hypervisor: ${{ matrix.hypervisor }}
+ cpu: ${{ matrix.cpu }}
+
+ # Combine benchmark reports into a single artifact for the hyperlight-gh-bot
+ # to post as a PR comment. Only runs for PRs (not merge groups) with code changes.
+ benchmark-comment:
+ name: post-benchmark-comment
+ needs:
+ - docs-pr
+ - benchmarks
+ if: ${{ !cancelled() && !failure() && needs.docs-pr.outputs.docs-only == 'false' && github.event_name == 'pull_request' }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Download benchmark reports
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: benchmark-report_*
+ path: reports/
+
+ - name: Combine benchmark reports
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const fs = require('fs');
+ const path = require('path');
+
+ const reportsDir = 'reports';
+ if (!fs.existsSync(reportsDir)) {
+ console.log('No benchmark reports found, skipping.');
+ return;
+ }
+
+ const sections = [];
+ const dirs = fs.readdirSync(reportsDir).sort();
+ for (const dir of dirs) {
+ const mdPath = path.join(reportsDir, dir, 'benchmark.md');
+ if (!fs.existsSync(mdPath)) continue;
+
+ // Extract hypervisor/cpu from artifact name: benchmark-report_OS_hypervisor_cpu
+ const parts = dir.replace('benchmark-report_', '').split('_');
+ const os = parts[0];
+ const hypervisor = parts.slice(1, -1).join('_');
+ const cpu = parts[parts.length - 1];
+ const label = `${hypervisor} / ${cpu} (${os})`;
+
+ const content = fs.readFileSync(mdPath, 'utf8').trim();
+ sections.push(`\n${label}
\n\n${content}\n\n `);
+ }
+
+ if (sections.length === 0) {
+ console.log('No benchmark report content found, skipping.');
+ return;
+ }
+
+ const body = `## Benchmark Results\n\n${sections.join('\n\n')}`;
+ fs.writeFileSync('pr-comment.md', body);
+
+ - name: Upload PR comment artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: pr-comment
+ path: pr-comment.md
+ if-no-files-found: warn
+ retention-days: 1
+
spelling:
name: spell check with typos
runs-on: ubuntu-latest
@@ -167,6 +250,7 @@ jobs:
- build-test
- run-examples
- fuzzing
+ - benchmarks
- spelling
- license-headers
if: always()
diff --git a/.github/workflows/dep_benchmarks.yml b/.github/workflows/dep_benchmarks.yml
index b0c47be766..598e89f9ea 100644
--- a/.github/workflows/dep_benchmarks.yml
+++ b/.github/workflows/dep_benchmarks.yml
@@ -56,7 +56,6 @@ on:
required: false
type: number
default: 5
-
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: full
@@ -135,6 +134,16 @@ jobs:
- name: Run benchmarks
run: just bench-ci main
+ - name: Create benchmarks report
+ run: cargo ci bench-report > target/criterion/benchmark.md
+
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: benchmark-report_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}
+ path: target/criterion/benchmark.md
+ if-no-files-found: error
+ retention-days: 1
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}
diff --git a/Cargo.lock b/Cargo.lock
index ba73df16d4..67e3f6dff5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -58,18 +58,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
-name = "anstream"
-version = "0.6.21"
+name = "ansi-replace"
+version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
+checksum = "7f8b155ab93213f41c886d3a46e335258428e52c7cf868e25cf099d50274496d"
dependencies = [
- "anstyle",
- "anstyle-parse 0.2.7",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
+ "regex",
+ "stable-pattern",
]
[[package]]
@@ -79,7 +74,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
- "anstyle-parse 1.0.0",
+ "anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
@@ -93,15 +88,6 @@ version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
-[[package]]
-name = "anstyle-parse"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
-dependencies = [
- "utf8parse",
-]
-
[[package]]
name = "anstyle-parse"
version = "1.0.0"
@@ -117,7 +103,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -128,7 +114,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -499,25 +485,38 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.58"
+version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806"
+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
+ "clap_derive",
]
[[package]]
name = "clap_builder"
-version = "4.5.58"
+version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2"
+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
- "anstream 0.6.21",
+ "anstream",
"anstyle",
"clap_lex",
"strsim",
]
+[[package]]
+name = "clap_derive"
+version = "4.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "clap_lex"
version = "1.0.0"
@@ -530,6 +529,19 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "constant_time_eq"
version = "0.4.2"
@@ -609,6 +621,19 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "cpu-pin"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb5bc1be026f7f066429ce0611e23a341db91b91ed701de4a1432d01d3ed1105"
+dependencies = [
+ "libc",
+ "mach2 0.6.0",
+ "once_cell",
+ "tokio",
+ "windows",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -661,6 +686,17 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "criterion-markdown"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76c4282b4c193d947d82edbbc558d3c0bb8a538c6a7040169fd500900ee4443"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "criterion-plot"
version = "0.8.2"
@@ -671,6 +707,23 @@ dependencies = [
"itertools 0.13.0",
]
+[[package]]
+name = "criterion-swarm"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82693ab3e80479ca52770515f3837028800e365aac48a83edb86f920a6783232"
+dependencies = [
+ "ansi-replace",
+ "anyhow",
+ "clap",
+ "cpu-pin",
+ "indicatif",
+ "regex",
+ "serde_json",
+ "simple-pool",
+ "tokio",
+]
+
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
@@ -769,7 +822,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -815,6 +868,12 @@ dependencies = [
"zerocopy",
]
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "endian-type"
version = "0.1.2"
@@ -837,7 +896,7 @@ version = "0.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
dependencies = [
- "anstream 1.0.0",
+ "anstream",
"anstyle",
"env_filter",
"jiff",
@@ -857,7 +916,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -1190,7 +1249,7 @@ dependencies = [
"gobject-sys",
"libc",
"system-deps",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -1456,6 +1515,19 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "hyperlight-ci"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "criterion-markdown",
+ "criterion-swarm",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
[[package]]
name = "hyperlight-common"
version = "0.15.0"
@@ -1626,7 +1698,7 @@ dependencies = [
"vmm-sys-util",
"windows",
"windows-result",
- "windows-sys",
+ "windows-sys 0.61.2",
"windows-version",
]
@@ -1820,6 +1892,19 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "indicatif"
+version = "0.17.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+dependencies = [
+ "console",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+ "web-time",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -2105,6 +2190,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "mach2"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b"
+
[[package]]
name = "macho-unwind-info"
version = "0.5.0"
@@ -2216,7 +2307,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
dependencies = [
"libc",
"wasi",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2280,7 +2371,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2313,6 +2404,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
[[package]]
name = "object"
version = "0.39.0"
@@ -2324,6 +2421,12 @@ dependencies = [
"ruzstd",
]
+[[package]]
+name = "object-id"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c587bd1cd63959a8520442afc0f92a875d83deea175c7b48dd9f104a2c5070a9"
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -2785,7 +2888,7 @@ dependencies = [
"bindgen 0.70.1",
"libc",
"libproc",
- "mach2",
+ "mach2 0.4.3",
"winapi",
]
@@ -3152,7 +3255,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3363,6 +3466,16 @@ version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+[[package]]
+name = "simple-pool"
+version = "0.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "073382259dbeb56c3eaab04a1d330459f6490d1e518b2a8ee441c8bd00dbc092"
+dependencies = [
+ "object-id",
+ "parking_lot",
+]
+
[[package]]
name = "sketches-ddsketch"
version = "0.3.0"
@@ -3388,7 +3501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3406,6 +3519,15 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "stable-pattern"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -3478,7 +3600,7 @@ dependencies = [
"getrandom 0.4.1",
"once_cell",
"rustix",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3553,7 +3675,7 @@ dependencies = [
"signal-hook-registry",
"socket2",
"tokio-macros",
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3913,6 +4035,12 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+[[package]]
+name = "unicode-width"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
+
[[package]]
name = "unicode-xid"
version = "0.2.6"
@@ -4195,7 +4323,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4305,6 +4433,15 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
[[package]]
name = "windows-sys"
version = "0.61.2"
@@ -4314,6 +4451,22 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
[[package]]
name = "windows-threading"
version = "0.2.1"
@@ -4332,6 +4485,54 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
[[package]]
name = "winnow"
version = "0.7.14"
diff --git a/Cargo.toml b/Cargo.toml
index e9b69f40de..6650dcbf99 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,7 @@ default-members = [
"src/hyperlight_testing",
]
members = [
+ "src/hyperlight_ci",
"src/hyperlight_common",
"src/hyperlight_guest",
"src/hyperlight_host",
diff --git a/Justfile b/Justfile
index 401897425f..bc8a0ee77b 100644
--- a/Justfile
+++ b/Justfile
@@ -401,12 +401,10 @@ bench-download os hypervisor cpu tag="":
# Warning: compares to and then OVERWRITES the given baseline
bench-ci baseline features="":
- @# Benchmarks are always run with release builds for meaningful results
- cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+ cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }}
bench features="":
- @# Benchmarks are always run with release builds for meaningful results
- cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+ cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose
###############
### FUZZING ###
diff --git a/src/hyperlight_ci/Cargo.toml b/src/hyperlight_ci/Cargo.toml
new file mode 100644
index 0000000000..635ea72cd7
--- /dev/null
+++ b/src/hyperlight_ci/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "hyperlight-ci"
+edition = "2021"
+# fields intentionally not set, to avoid accidentally publishing this crate to crates.io
+description = """
+Hyperlight's CI and development tools.
+"""
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = "1"
+clap = { version = "4.6.1", features = ["derive"] }
+criterion-markdown = "0.1"
+criterion-swarm = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tokio = { version = "1", features = ["rt"] }
\ No newline at end of file
diff --git a/src/hyperlight_ci/src/bench.rs b/src/hyperlight_ci/src/bench.rs
new file mode 100644
index 0000000000..f819b7a8c6
--- /dev/null
+++ b/src/hyperlight_ci/src/bench.rs
@@ -0,0 +1,119 @@
+/*
+Copyright 2025 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+//! The `bench` subcommand: runs criterion benchmarks in parallel via criterion-swarm.
+
+use std::path::PathBuf;
+
+use anyhow::Context;
+use criterion_swarm::{CriterionSwarm, OutputMode};
+
+/// An output mode flag for `--build-output` / `--benchmarks-output`.
+#[derive(Clone, Debug)]
+pub(crate) struct OutputModeFlags(OutputMode);
+
+impl OutputModeFlags {
+ /// Parse a single token into an `OutputMode` flag.
+ fn parse_one(s: &str) -> Result {
+ match s.trim().to_ascii_lowercase().as_str() {
+ "spinner" => Ok(OutputMode::SPINNER),
+ "stream" => Ok(OutputMode::STREAM),
+ "summary" => Ok(OutputMode::SUMMARY),
+ "none" | "silent" => Ok(OutputMode::SILENT),
+ other => Err(format!(
+ "unknown output mode `{other}` (expected: spinner, stream, summary, none)"
+ )),
+ }
+ }
+}
+
+impl std::str::FromStr for OutputModeFlags {
+ type Err = String;
+ fn from_str(s: &str) -> Result {
+ let mut mode = OutputMode::SILENT;
+ for part in s.split(',') {
+ mode = mode | Self::parse_one(part)?;
+ }
+ Ok(Self(mode))
+ }
+}
+
+/// Merge a `Vec` into a single `OutputMode` by OR-ing them together.
+fn merge_output_modes(flags: &[OutputModeFlags]) -> OutputMode {
+ flags.iter().fold(OutputMode::SILENT, |acc, f| acc | f.0)
+}
+
+/// Command-line arguments for the `bench` subcommand.
+#[derive(clap::Args)]
+pub struct BenchArgs {
+ /// Pre-built benchmark binary to use (skip build step; can be specified multiple times)
+ #[arg(long)]
+ pub binary: Vec,
+
+ /// Number of benchmarks to run in parallel (0 = all P-cores, default: 0)
+ #[arg(long, short, default_value_t = 0)]
+ pub jobs: usize,
+
+ /// Build output mode (comma-separated or repeated): spinner, stream, summary, none
+ #[arg(long, value_delimiter = ',', default_value = "spinner,stream,summary")]
+ pub build_output: Vec,
+
+ /// Benchmarks output mode (comma-separated or repeated): spinner, stream, summary, none
+ #[arg(long, value_delimiter = ',', default_value = "spinner,stream,summary")]
+ pub benchmarks_output: Vec,
+
+ /// Additional features to pass to cargo when building benchmarks (can be specified multiple times)
+ #[arg(short = 'F', long)]
+ pub features: Vec,
+
+ /// Additional arguments to forward to criterion benchmarks
+ #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
+ pub bench_args: Vec,
+}
+
+pub async fn run(args: BenchArgs) -> anyhow::Result<()> {
+ let mut swarm = CriterionSwarm::builder().jobs(args.jobs);
+
+ if !args.binary.is_empty() {
+ swarm = swarm.binaries(args.binary);
+ }
+
+ if !args.features.is_empty() {
+ swarm = swarm.build_args(["--features".to_string(), args.features.join(",")]);
+ }
+
+ for arg in args.bench_args {
+ swarm = swarm.bench_arg(arg);
+ }
+
+ let build_mode = merge_output_modes(&args.build_output);
+ let bench_mode = merge_output_modes(&args.benchmarks_output);
+ swarm = swarm.output(
+ criterion_swarm::ProgressReporter::new()
+ .build(build_mode)
+ .benchmarks(bench_mode),
+ );
+
+ let swarm = swarm
+ .prepare()
+ .await
+ .context("Failed to prepare criterion swarm")?;
+ if bench_mode == (bench_mode | OutputMode::SUMMARY) {
+ let total = swarm.benchmarks().len();
+ let jobs = swarm.jobs().min(total);
+ println!("Running {total} benchmarks with parallelism {jobs}");
+ }
+ swarm.run().await.context("Failed to run criterion swarm")
+}
diff --git a/src/hyperlight_ci/src/bench_report.rs b/src/hyperlight_ci/src/bench_report.rs
new file mode 100644
index 0000000000..0abedb0c9c
--- /dev/null
+++ b/src/hyperlight_ci/src/bench_report.rs
@@ -0,0 +1,79 @@
+/*
+Copyright 2025 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+//! The `bench-report` subcommand: generates a markdown table from existing
+//! criterion benchmark results in `target/criterion/`.
+
+use std::path::PathBuf;
+
+use anyhow::{Context, Result};
+use clap::Args;
+use criterion_swarm::{CriterionSwarm, NoopReporter};
+
+/// Command-line arguments for the `bench-report` subcommand.
+#[derive(Args)]
+pub struct BenchReportArgs {
+ /// Benchmark binary to list benchmarks from (can be specified multiple times).
+ /// When provided, only benchmarks available in these binaries are included.
+ #[arg(long)]
+ pub binary: Vec,
+
+ /// Path to the criterion output directory
+ #[arg(long, default_value = "target/criterion")]
+ pub criterion_dir: PathBuf,
+
+ /// Additional arguments to forward to criterion benchmarks (e.g. filter, --exact)
+ #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
+ pub bench_args: Vec,
+}
+
+/// Entry point for the bench-report subcommand.
+pub async fn run(args: BenchReportArgs) -> Result<()> {
+ let allowlist = build_allowlist(&args).await?;
+
+ let markdown = criterion_markdown::render(&args.criterion_dir, &allowlist)?;
+
+ print!("{markdown}");
+
+ Ok(())
+}
+
+/// Builds an allowlist of benchmark full_ids by discovering benchmarks via CriterionSwarm.
+///
+/// All trailing arguments (filter, --exact, etc.) are forwarded as bench args
+/// to CriterionSwarm so it handles filtering during discovery.
+async fn build_allowlist(args: &BenchReportArgs) -> Result> {
+ let mut swarm = CriterionSwarm::builder();
+
+ if !args.binary.is_empty() {
+ swarm = swarm.binaries(&args.binary);
+ }
+
+ for arg in &args.bench_args {
+ swarm = swarm.bench_arg(arg);
+ }
+
+ let discovered = swarm
+ .output(NoopReporter)
+ .prepare()
+ .await
+ .context("Failed to discover benchmarks")?;
+
+ Ok(discovered
+ .benchmarks()
+ .into_iter()
+ .map(str::to_string)
+ .collect())
+}
diff --git a/src/hyperlight_ci/src/main.rs b/src/hyperlight_ci/src/main.rs
new file mode 100644
index 0000000000..38e77e85d2
--- /dev/null
+++ b/src/hyperlight_ci/src/main.rs
@@ -0,0 +1,46 @@
+/*
+Copyright 2025 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+mod bench;
+mod bench_report;
+
+use clap::{Parser, Subcommand};
+
+#[derive(Parser)]
+#[command(
+ name = "hyperlight-ci",
+ about = "Hyperlight's CI and development tools"
+)]
+struct Cli {
+ #[command(subcommand)]
+ command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+ /// Run benchmarks using the benchmark binary directly
+ Bench(bench::BenchArgs),
+ /// Generate a markdown table from existing criterion benchmark results
+ BenchReport(bench_report::BenchReportArgs),
+}
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+ let cli = Cli::parse();
+ match cli.command {
+ Commands::Bench(args) => bench::run(args).await,
+ Commands::BenchReport(args) => bench_report::run(args).await,
+ }
+}